From aa14a7c708411c7af936dedda33b72b330d7efb1 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Thu, 12 Jul 2018 21:18:15 +0200 Subject: [PATCH 001/109] Removed unmerged feature from changelog --- CHANGELOG.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c6178733..be560d23 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,7 +6,6 @@ + [TESTS] replace nsot container with requests-mock #172 + [PLUGIN_IMPROVEMENT] Support SSH Agent forwarding for paramiko SSH connections #159 + [PLUGIN_IMPROVEMENT] allow passing options to napalm getters #156 -+ [CORE_NEW_FEATURE] Filter object #155 + [PLUGIN_BUGFIX] Fix for SSH and API port mapping issues #154 + [CORE_NEW_FEATURE] add to_dict function so the inventory is serializable #146 + [CORE_BUGFIX] Fix issues with using built-in and overwriting variable with loop variable #144 From aec37d22786577ad8eb8ab6d608f70ea06d4afa1 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Tue, 17 Jul 2018 11:00:54 +0200 Subject: [PATCH 002/109] emove support for Contact your system administrator',) -^^^^ END read_data ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/tests/plugins/functions/text/output_data/multiple_tasks_python27.stderr b/tests/plugins/functions/text/output_data/multiple_tasks_python27.stderr deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/plugins/functions/text/output_data/multiple_tasks_python27.stdout b/tests/plugins/functions/text/output_data/multiple_tasks_python27.stdout deleted file mode 100644 index 5c1105d5..00000000 --- a/tests/plugins/functions/text/output_data/multiple_tasks_python27.stdout +++ /dev/null @@ -1,30 +0,0 @@ -**** Behold the data! ********************************************************** -data_with_greeting************************************************************** -* dev1.group_1 ** changed : False ********************************************** -vvvv data_with_greeting ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO ----- echo_task ** changed : False ---------------------------------------------- INFO -Hello from Nornir ----- load_data ** changed : False ---------------------------------------------- INFO -{ 'os': 'Linux', 'services': ['http', 'smtp', 'dns']} -^^^^ END data_with_greeting ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* dev2.group_1 ** changed : False ********************************************** -vvvv data_with_greeting ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO ----- echo_task ** changed : False ---------------------------------------------- INFO -Hello from Nornir ----- load_data ** changed : False ---------------------------------------------- INFO -{ 'os': 'Linux', 'services': ['http', 'smtp', 'dns']} -^^^^ END data_with_greeting ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* dev3.group_2 ** changed : False ********************************************** -vvvv data_with_greeting ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO ----- echo_task ** changed : False ---------------------------------------------- INFO -Hello from Nornir ----- load_data ** changed : False ---------------------------------------------- INFO -{ 'os': 'Linux', 'services': ['http', 'smtp', 'dns']} -^^^^ END data_with_greeting ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* dev4.group_2 ** changed : False ********************************************** -vvvv data_with_greeting ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO ----- echo_task ** changed : False ---------------------------------------------- INFO -Hello from Nornir ----- load_data ** changed : False ---------------------------------------------- INFO -{ 'os': 'Linux', 'services': ['http', 'smtp', 'dns']} -^^^^ END data_with_greeting ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/tests/plugins/tasks/data/test_load_json.py b/tests/plugins/tasks/data/test_load_json.py index 686952fd..95d8d9e3 100644 --- a/tests/plugins/tasks/data/test_load_json.py +++ b/tests/plugins/tasks/data/test_load_json.py @@ -1,5 +1,4 @@ import os -import sys from nornir.plugins.tasks import data @@ -30,15 +29,10 @@ def test_load_json_error_broken_file(self, nornir): def test_load_json_error_missing_file(self, nornir): test_file = "{}/missing.json".format(data_dir) - if sys.version_info.major == 2: - not_found = IOError - else: - not_found = FileNotFoundError # noqa - results = nornir.run(data.load_json, file=test_file) processed = False for result in results.values(): processed = True - assert isinstance(result.exception, not_found) + assert isinstance(result.exception, FileNotFoundError) assert processed nornir.data.reset_failed_hosts() diff --git a/tests/plugins/tasks/data/test_load_yaml.py b/tests/plugins/tasks/data/test_load_yaml.py index de9eba05..d11b473b 100644 --- a/tests/plugins/tasks/data/test_load_yaml.py +++ b/tests/plugins/tasks/data/test_load_yaml.py @@ -1,5 +1,4 @@ import os -import sys from nornir.plugins.tasks import data @@ -34,16 +33,10 @@ def test_load_yaml_error_broken_file(self, nornir): def test_load_yaml_error_missing_file(self, nornir): test_file = "{}/missing.yaml".format(data_dir) - - if sys.version_info.major == 2: - not_found = IOError - else: - not_found = FileNotFoundError # noqa - results = nornir.run(data.load_yaml, file=test_file) processed = False for result in results.values(): processed = True - assert isinstance(result.exception, not_found) + assert isinstance(result.exception, FileNotFoundError) assert processed nornir.data.reset_failed_hosts() diff --git a/tests/wrapper.py b/tests/wrapper.py index 66b83da1..2378485e 100644 --- a/tests/wrapper.py +++ b/tests/wrapper.py @@ -1,10 +1,7 @@ import sys -from decorator import decorator +from io import StringIO -if sys.version_info.major == 2: - from StringIO import StringIO -else: - from io import StringIO +from decorator import decorator def wrap_cli_test(output, save_output=False): @@ -33,8 +30,6 @@ def run_test(func, *args, **kwargs): sys.stderr = backup_stderr output_file = output - if sys.version_info.major == 2: - output_file += "_python27" if save_output: with open("{}.stdout".format(output_file), "w+") as f: diff --git a/tox.ini b/tox.ini index b26595fb..0249d564 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py34,py35,py36 +envlist = py36,py37 [testenv] deps = From d9fd9a290477072141e381299a246aa26b05e3bb Mon Sep 17 00:00:00 2001 From: David Barroso Date: Tue, 17 Jul 2018 11:23:55 +0200 Subject: [PATCH 003/109] Migrate to ruamel.yaml and add `ordered_dict` parameter to `load_json` and `load_yaml` plugins (#157) --- nornir/core/configuration.py | 4 ++-- nornir/plugins/inventory/simple.py | 6 +++--- nornir/plugins/tasks/data/load_json.py | 17 +++++++++++++++-- nornir/plugins/tasks/data/load_yaml.py | 19 +++++++++++++++---- requirements.txt | 1 - tests/core/test_configuration/config.yaml | 2 +- .../plugins/tasks/data/test_data/simple.json | 3 ++- .../plugins/tasks/data/test_data/simple.yaml | 3 +++ tests/plugins/tasks/data/test_load_json.py | 12 ++++++++++++ tests/plugins/tasks/data/test_load_yaml.py | 15 +++++++++++++-- 10 files changed, 66 insertions(+), 16 deletions(-) diff --git a/nornir/core/configuration.py b/nornir/core/configuration.py index a691343c..aa4d5f74 100644 --- a/nornir/core/configuration.py +++ b/nornir/core/configuration.py @@ -2,7 +2,7 @@ import os -import yaml +import ruamel.yaml CONF = { @@ -93,7 +93,7 @@ class Config(object): def __init__(self, config_file=None, **kwargs): if config_file: with open(config_file, "r") as f: - data = yaml.load(f.read()) or {} + data = ruamel.yaml.safe_load(f.read()) or {} else: data = {} diff --git a/nornir/plugins/inventory/simple.py b/nornir/plugins/inventory/simple.py index cf77c98a..03b87039 100644 --- a/nornir/plugins/inventory/simple.py +++ b/nornir/plugins/inventory/simple.py @@ -4,7 +4,7 @@ from nornir.core.inventory import Inventory -import yaml +import ruamel.yaml class SimpleInventory(Inventory): @@ -118,12 +118,12 @@ class SimpleInventory(Inventory): def __init__(self, host_file="hosts.yaml", group_file="groups.yaml", **kwargs): with open(host_file, "r") as f: - hosts = yaml.load(f.read()) + hosts = ruamel.yaml.safe_load(f.read()) if group_file: if os.path.exists(group_file): with open(group_file, "r") as f: - groups = yaml.load(f.read()) + groups = ruamel.yaml.safe_load(f.read()) else: logging.warning("{}: doesn't exist".format(group_file)) groups = {} diff --git a/nornir/plugins/tasks/data/load_json.py b/nornir/plugins/tasks/data/load_json.py index 06782000..453e9f6f 100644 --- a/nornir/plugins/tasks/data/load_json.py +++ b/nornir/plugins/tasks/data/load_json.py @@ -1,20 +1,33 @@ +import collections import json from nornir.core.task import Result -def load_json(task, file): +def load_json(task, file, ordered_dict=False): """ Loads a json file. Arguments: file (str): path to the file containing the json file to load + ordered_dict (bool): If set to true used OrderedDict to load maps + + Examples: + + Simple example with ``ordered_dict``:: + + > nr.run(task=load_json, + file="mydata.json", + ordered_dict=True) Returns: :obj:`nornir.core.task.Result`: * result (``dict``): dictionary with the contents of the file """ + kwargs = {} + if ordered_dict: + kwargs["object_pairs_hook"] = collections.OrderedDict with open(file, "r") as f: - data = json.loads(f.read()) + data = json.loads(f.read(), **kwargs) return Result(host=task.host, result=data) diff --git a/nornir/plugins/tasks/data/load_yaml.py b/nornir/plugins/tasks/data/load_yaml.py index 2e989097..52502419 100644 --- a/nornir/plugins/tasks/data/load_yaml.py +++ b/nornir/plugins/tasks/data/load_yaml.py @@ -1,21 +1,32 @@ from nornir.core.task import Result +import ruamel.yaml -import yaml - -def load_yaml(task, file): +def load_yaml(task, file, ordered_dict=False): """ Loads a yaml file. Arguments: file (str): path to the file containing the yaml file to load + ordered_dict (bool): If set to true used OrderedDict to load maps + + Examples: + + Simple example with ``ordered_dict``:: + + > nr.run(task=load_yaml, + file="mydata.yaml", + ordered_dict=True) Returns: :obj:`nornir.core.task.Result`: * result (``dict``): dictionary with the contents of the file """ + kwargs = {} + kwargs["typ"] = "rt" if ordered_dict else "safe" with open(file, "r") as f: - data = yaml.load(f.read()) + yml = ruamel.yaml.YAML(pure=True, **kwargs) + data = yml.load(f.read()) return Result(host=task.host, result=data) diff --git a/requirements.txt b/requirements.txt index 12396b0b..335ed1cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ colorama -pyyaml jinja2 napalm>=2.3.0 netmiko>=2.1.1 diff --git a/tests/core/test_configuration/config.yaml b/tests/core/test_configuration/config.yaml index 164c4f87..e66cb710 100644 --- a/tests/core/test_configuration/config.yaml +++ b/tests/core/test_configuration/config.yaml @@ -1,6 +1,6 @@ --- num_workers: 10 -raise_on_error: no +raise_on_error: false user_defined: "asdasd" my_root: user_defined: "i am nested" diff --git a/tests/plugins/tasks/data/test_data/simple.json b/tests/plugins/tasks/data/test_data/simple.json index 3a17c39d..750df624 100644 --- a/tests/plugins/tasks/data/test_data/simple.json +++ b/tests/plugins/tasks/data/test_data/simple.json @@ -3,5 +3,6 @@ "services": [ "dhcp", "dns" - ] + ], + "a_dict": {"a": 1, "b": 2} } diff --git a/tests/plugins/tasks/data/test_data/simple.yaml b/tests/plugins/tasks/data/test_data/simple.yaml index fe7fdb00..3e9971f7 100644 --- a/tests/plugins/tasks/data/test_data/simple.yaml +++ b/tests/plugins/tasks/data/test_data/simple.yaml @@ -3,3 +3,6 @@ env: test services: - dhcp - dns +a_dict: + a: 1 + b: 2 diff --git a/tests/plugins/tasks/data/test_load_json.py b/tests/plugins/tasks/data/test_load_json.py index 95d8d9e3..4155478b 100644 --- a/tests/plugins/tasks/data/test_load_json.py +++ b/tests/plugins/tasks/data/test_load_json.py @@ -1,4 +1,5 @@ import os +from collections import OrderedDict from nornir.plugins.tasks import data @@ -16,6 +17,17 @@ def test_load_json(self, nornir): d = r.result assert d["env"] == "test" assert d["services"] == ["dhcp", "dns"] + assert isinstance(d["a_dict"], dict) + + def test_load_json_ordered_dict(self, nornir): + test_file = "{}/simple.json".format(data_dir) + result = nornir.run(data.load_json, file=test_file, ordered_dict=True) + + for h, r in result.items(): + d = r.result + assert d["env"] == "test" + assert d["services"] == ["dhcp", "dns"] + assert isinstance(d["a_dict"], OrderedDict) def test_load_json_error_broken_file(self, nornir): test_file = "{}/broken.json".format(data_dir) diff --git a/tests/plugins/tasks/data/test_load_yaml.py b/tests/plugins/tasks/data/test_load_yaml.py index d11b473b..9f24d6e2 100644 --- a/tests/plugins/tasks/data/test_load_yaml.py +++ b/tests/plugins/tasks/data/test_load_yaml.py @@ -1,10 +1,10 @@ import os - +from collections import OrderedDict from nornir.plugins.tasks import data -from yaml.scanner import ScannerError +from ruamel.yaml.scanner import ScannerError data_dir = "{}/test_data".format(os.path.dirname(os.path.realpath(__file__))) @@ -20,6 +20,17 @@ def test_load_yaml(self, nornir): d = r.result assert d["env"] == "test" assert d["services"] == ["dhcp", "dns"] + assert isinstance(d["a_dict"], dict) + + def test_load_yaml_ordered_dict(self, nornir): + test_file = "{}/simple.yaml".format(data_dir) + result = nornir.run(data.load_yaml, file=test_file, ordered_dict=True) + + for h, r in result.items(): + d = r.result + assert d["env"] == "test" + assert d["services"] == ["dhcp", "dns"] + assert isinstance(d["a_dict"], OrderedDict) def test_load_yaml_error_broken_file(self, nornir): test_file = "{}/broken.yaml".format(data_dir) From 78758d024500c64fc71414c6423da97b4f8cad05 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Tue, 17 Jul 2018 12:06:47 +0200 Subject: [PATCH 004/109] Filter object (#176) --- docs/tutorials/intro/inventory.ipynb | 382 +++++++++++++++----- docs/tutorials/intro/inventory/hosts.yaml | 12 + nornir/core/__init__.py | 4 +- nornir/core/filter.py | 88 +++++ nornir/core/inventory.py | 18 +- tests/core/test_filter.py | 114 ++++++ tests/core/test_inventory.py | 13 +- tests/inventory_data/groups.yaml | 5 + tests/inventory_data/hosts.yaml | 12 + tests/plugins/inventory/helpers/__init__.py | 0 10 files changed, 542 insertions(+), 106 deletions(-) create mode 100644 nornir/core/filter.py create mode 100644 tests/core/test_filter.py create mode 100644 tests/plugins/inventory/helpers/__init__.py diff --git a/docs/tutorials/intro/inventory.ipynb b/docs/tutorials/intro/inventory.ipynb index 24611c5f..8bbadabb 100644 --- a/docs/tutorials/intro/inventory.ipynb +++ b/docs/tutorials/intro/inventory.ipynb @@ -117,132 +117,144 @@ " 10 - cmh\n", " 11 nornir_nos: linux\n", " 12 type: host\n", - " 13 \n", - " 14 host2.cmh:\n", - " 15 nornir_host: 127.0.0.1\n", - " 16 nornir_ssh_port: 2202\n", - " 17 nornir_username: vagrant\n", - " 18 nornir_password: vagrant\n", - " 19 site: cmh\n", - " 20 role: host\n", - " 21 groups:\n", - " 22 - cmh\n", - " 23 nornir_nos: linux\n", - " 24 type: host\n", - " 25 \n", - " 26 spine00.cmh:\n", - " 27 nornir_host: 127.0.0.1\n", - " 28 nornir_username: vagrant\n", - " 29 nornir_password: vagrant\n", - " 30 nornir_network_api_port: 12444\n", - " 31 site: cmh\n", - " 32 role: spine\n", - " 33 groups:\n", - " 34 - cmh\n", - " 35 nornir_nos: eos\n", - " 36 type: network_device\n", + " 13 nested_data:\n", + " 14 a_dict:\n", + " 15 a: 1\n", + " 16 b: 2\n", + " 17 a_list: [1, 2]\n", + " 18 a_string: "asdasd"\n", + " 19 \n", + " 20 host2.cmh:\n", + " 21 nornir_host: 127.0.0.1\n", + " 22 nornir_ssh_port: 2202\n", + " 23 nornir_username: vagrant\n", + " 24 nornir_password: vagrant\n", + " 25 site: cmh\n", + " 26 role: host\n", + " 27 groups:\n", + " 28 - cmh\n", + " 29 nornir_nos: linux\n", + " 30 type: host\n", + " 31 nested_data:\n", + " 32 a_dict:\n", + " 33 b: 2\n", + " 34 c: 3\n", + " 35 a_list: [1, 2]\n", + " 36 a_string: "qwe"\n", " 37 \n", - " 38 spine01.cmh:\n", + " 38 spine00.cmh:\n", " 39 nornir_host: 127.0.0.1\n", " 40 nornir_username: vagrant\n", - " 41 nornir_password: ""\n", - " 42 nornir_network_api_port: 12204\n", + " 41 nornir_password: vagrant\n", + " 42 nornir_network_api_port: 12444\n", " 43 site: cmh\n", " 44 role: spine\n", " 45 groups:\n", " 46 - cmh\n", - " 47 nornir_nos: junos\n", + " 47 nornir_nos: eos\n", " 48 type: network_device\n", " 49 \n", - " 50 leaf00.cmh:\n", + " 50 spine01.cmh:\n", " 51 nornir_host: 127.0.0.1\n", " 52 nornir_username: vagrant\n", - " 53 nornir_password: vagrant\n", - " 54 nornir_network_api_port: 12443\n", + " 53 nornir_password: ""\n", + " 54 nornir_network_api_port: 12204\n", " 55 site: cmh\n", - " 56 role: leaf\n", + " 56 role: spine\n", " 57 groups:\n", " 58 - cmh\n", - " 59 nornir_nos: eos\n", + " 59 nornir_nos: junos\n", " 60 type: network_device\n", - " 61 asn: 65100\n", - " 62 \n", - " 63 leaf01.cmh:\n", - " 64 nornir_host: 127.0.0.1\n", - " 65 nornir_username: vagrant\n", - " 66 nornir_password: ""\n", - " 67 nornir_network_api_port: 12203\n", - " 68 site: cmh\n", - " 69 role: leaf\n", - " 70 groups:\n", - " 71 - cmh\n", - " 72 nornir_nos: junos\n", - " 73 type: network_device\n", - " 74 asn: 65101\n", - " 75 \n", - " 76 host1.bma:\n", - " 77 site: bma\n", - " 78 role: host\n", - " 79 groups:\n", - " 80 - bma\n", - " 81 nornir_nos: linux\n", - " 82 type: host\n", - " 83 \n", - " 84 host2.bma:\n", - " 85 site: bma\n", - " 86 role: host\n", - " 87 groups:\n", - " 88 - bma\n", - " 89 nornir_nos: linux\n", - " 90 type: host\n", - " 91 \n", - " 92 spine00.bma:\n", - " 93 nornir_host: 127.0.0.1\n", - " 94 nornir_username: vagrant\n", - " 95 nornir_password: vagrant\n", - " 96 nornir_network_api_port: 12444\n", + " 61 \n", + " 62 leaf00.cmh:\n", + " 63 nornir_host: 127.0.0.1\n", + " 64 nornir_username: vagrant\n", + " 65 nornir_password: vagrant\n", + " 66 nornir_network_api_port: 12443\n", + " 67 site: cmh\n", + " 68 role: leaf\n", + " 69 groups:\n", + " 70 - cmh\n", + " 71 nornir_nos: eos\n", + " 72 type: network_device\n", + " 73 asn: 65100\n", + " 74 \n", + " 75 leaf01.cmh:\n", + " 76 nornir_host: 127.0.0.1\n", + " 77 nornir_username: vagrant\n", + " 78 nornir_password: ""\n", + " 79 nornir_network_api_port: 12203\n", + " 80 site: cmh\n", + " 81 role: leaf\n", + " 82 groups:\n", + " 83 - cmh\n", + " 84 nornir_nos: junos\n", + " 85 type: network_device\n", + " 86 asn: 65101\n", + " 87 \n", + " 88 host1.bma:\n", + " 89 site: bma\n", + " 90 role: host\n", + " 91 groups:\n", + " 92 - bma\n", + " 93 nornir_nos: linux\n", + " 94 type: host\n", + " 95 \n", + " 96 host2.bma:\n", " 97 site: bma\n", - " 98 role: spine\n", + " 98 role: host\n", " 99 groups:\n", "100 - bma\n", - "101 nornir_nos: eos\n", - "102 type: network_device\n", + "101 nornir_nos: linux\n", + "102 type: host\n", "103 \n", - "104 spine01.bma:\n", + "104 spine00.bma:\n", "105 nornir_host: 127.0.0.1\n", "106 nornir_username: vagrant\n", - "107 nornir_password: ""\n", - "108 nornir_network_api_port: 12204\n", + "107 nornir_password: vagrant\n", + "108 nornir_network_api_port: 12444\n", "109 site: bma\n", "110 role: spine\n", "111 groups:\n", "112 - bma\n", - "113 nornir_nos: junos\n", + "113 nornir_nos: eos\n", "114 type: network_device\n", "115 \n", - "116 leaf00.bma:\n", + "116 spine01.bma:\n", "117 nornir_host: 127.0.0.1\n", "118 nornir_username: vagrant\n", - "119 nornir_password: vagrant\n", - "120 nornir_network_api_port: 12443\n", + "119 nornir_password: ""\n", + "120 nornir_network_api_port: 12204\n", "121 site: bma\n", - "122 role: leaf\n", + "122 role: spine\n", "123 groups:\n", "124 - bma\n", - "125 nornir_nos: eos\n", + "125 nornir_nos: junos\n", "126 type: network_device\n", "127 \n", - "128 leaf01.bma:\n", + "128 leaf00.bma:\n", "129 nornir_host: 127.0.0.1\n", "130 nornir_username: vagrant\n", - "131 nornir_password: wrong_password\n", - "132 nornir_network_api_port: 12203\n", + "131 nornir_password: vagrant\n", + "132 nornir_network_api_port: 12443\n", "133 site: bma\n", "134 role: leaf\n", "135 groups:\n", "136 - bma\n", - "137 nornir_nos: junos\n", + "137 nornir_nos: eos\n", "138 type: network_device\n", + "139 \n", + "140 leaf01.bma:\n", + "141 nornir_host: 127.0.0.1\n", + "142 nornir_username: vagrant\n", + "143 nornir_password: wrong_password\n", + "144 nornir_network_api_port: 12203\n", + "145 site: bma\n", + "146 role: leaf\n", + "147 groups:\n", + "148 - bma\n", + "149 nornir_nos: junos\n", + "150 type: network_device\n", "\n", "\n" ], @@ -408,7 +420,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 4, @@ -438,18 +450,18 @@ { "data": { "text/plain": [ - "{'host1.bma': Host: host1.bma,\n", - " 'host1.cmh': Host: host1.cmh,\n", - " 'host2.bma': Host: host2.bma,\n", + "{'host1.cmh': Host: host1.cmh,\n", " 'host2.cmh': Host: host2.cmh,\n", - " 'leaf00.bma': Host: leaf00.bma,\n", + " 'spine00.cmh': Host: spine00.cmh,\n", + " 'spine01.cmh': Host: spine01.cmh,\n", " 'leaf00.cmh': Host: leaf00.cmh,\n", - " 'leaf01.bma': Host: leaf01.bma,\n", " 'leaf01.cmh': Host: leaf01.cmh,\n", + " 'host1.bma': Host: host1.bma,\n", + " 'host2.bma': Host: host2.bma,\n", " 'spine00.bma': Host: spine00.bma,\n", - " 'spine00.cmh': Host: spine00.cmh,\n", " 'spine01.bma': Host: spine01.bma,\n", - " 'spine01.cmh': Host: spine01.cmh}" + " 'leaf00.bma': Host: leaf00.bma,\n", + " 'leaf01.bma': Host: leaf01.bma}" ] }, "execution_count": 5, @@ -469,10 +481,10 @@ { "data": { "text/plain": [ - "{'bma': Group: bma,\n", - " 'cmh': Group: cmh,\n", + "{'global': Group: global,\n", " 'eu': Group: eu,\n", - " 'global': Group: global}" + " 'bma': Group: bma,\n", + " 'cmh': Group: cmh}" ] }, "execution_count": 6, @@ -965,10 +977,10 @@ "text/plain": [ "{'host1.bma': Host: host1.bma,\n", " 'host2.bma': Host: host2.bma,\n", - " 'leaf00.bma': Host: leaf00.bma,\n", - " 'leaf01.bma': Host: leaf01.bma,\n", " 'spine00.bma': Host: spine00.bma,\n", - " 'spine01.bma': Host: spine01.bma}" + " 'spine01.bma': Host: spine01.bma,\n", + " 'leaf00.bma': Host: leaf00.bma,\n", + " 'leaf01.bma': Host: leaf01.bma}" ] }, "execution_count": 21, @@ -986,7 +998,14 @@ "source": [ "#### Advanced filtering\n", "\n", - "Sometimes you need more fancy filtering. For those cases you can use a filter function:" + "Sometimes you need more fancy filtering. For those cases you have two options:\n", + "\n", + "1. Use a filter function.\n", + "2. Use a filter object.\n", + "\n", + "##### Filter functions\n", + "\n", + "The ``filter_func`` parameter let's you run your own code to filter the hosts. The function signature is as simple as ``my_func(host)`` where host is an object of type [Host](../../ref/api/inventory.rst#nornir.core.inventory.Host) and it has to return either ``True`` or ``False`` to indicate if you want to host or not." ] }, { @@ -1032,6 +1051,169 @@ "# Or a lambda function\n", "nr.filter(filter_func=lambda h: len(h.name) == 9).inventory.hosts.keys()" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Filter Object\n", + "\n", + "You can also use a filter object to create incrementally a complext query object. Let's see how it works by example:" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "# first you need to import the F object\n", + "from nornir.core.filter import F" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['host1.cmh', 'host2.cmh', 'spine00.cmh', 'spine01.cmh', 'leaf00.cmh', 'leaf01.cmh'])\n" + ] + } + ], + "source": [ + "# hosts in group cmh\n", + "cmh = nr.filter(F(groups__contains=\"cmh\"))\n", + "print(cmh.inventory.hosts.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['host1.cmh', 'host2.cmh', 'spine00.cmh', 'leaf00.cmh', 'host1.bma', 'host2.bma', 'spine00.bma', 'leaf00.bma'])\n" + ] + } + ], + "source": [ + "# devices running either linux or eos\n", + "linux_or_eos = nr.filter(F(nornir_nos=\"linux\") | F(nornir_nos=\"eos\"))\n", + "print(linux_or_eos.inventory.hosts.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['spine00.cmh', 'spine01.cmh'])\n" + ] + } + ], + "source": [ + "# spines in cmh\n", + "cmh_and_spine = nr.filter(F(groups__contains=\"cmh\") & F(role=\"spine\"))\n", + "print(cmh_and_spine.inventory.hosts.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['host1.cmh', 'host2.cmh', 'leaf00.cmh', 'leaf01.cmh'])\n" + ] + } + ], + "source": [ + "# cmh devices that are not spines\n", + "cmh_and_not_spine = nr.filter(F(groups__contains=\"cmh\") & ~F(role=\"spine\"))\n", + "print(cmh_and_not_spine.inventory.hosts.keys())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also access nested data and even check if dicts/lists/strings contains elements. Again, let's see by example:" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['host1.cmh'])\n" + ] + } + ], + "source": [ + "nested_string_asd = nr.filter(F(nested_data__a_string__contains=\"asd\"))\n", + "print(nested_string_asd.inventory.hosts.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['host2.cmh'])\n" + ] + } + ], + "source": [ + "a_dict_element_equals = nr.filter(F(nested_data__a_dict__c=3))\n", + "print(a_dict_element_equals.inventory.hosts.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['host1.cmh', 'host2.cmh'])\n" + ] + } + ], + "source": [ + "a_list_contains = nr.filter(F(nested_data__a_list__contains=2))\n", + "print(a_list_contains.inventory.hosts.keys())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can basically access any nested data by separating the elements in the path with two underscores `__`. Then you can use `__contains` to check if an element exists or if a string has a particular substring." + ] } ], "metadata": { diff --git a/docs/tutorials/intro/inventory/hosts.yaml b/docs/tutorials/intro/inventory/hosts.yaml index cc253298..d48d5c21 100644 --- a/docs/tutorials/intro/inventory/hosts.yaml +++ b/docs/tutorials/intro/inventory/hosts.yaml @@ -10,6 +10,12 @@ host1.cmh: - cmh nornir_nos: linux type: host + nested_data: + a_dict: + a: 1 + b: 2 + a_list: [1, 2] + a_string: "asdasd" host2.cmh: nornir_host: 127.0.0.1 @@ -22,6 +28,12 @@ host2.cmh: - cmh nornir_nos: linux type: host + nested_data: + a_dict: + b: 2 + c: 3 + a_list: [1, 2] + a_string: "qwe" spine00.cmh: nornir_host: 127.0.0.1 diff --git a/nornir/core/__init__.py b/nornir/core/__init__.py index 6905fb76..ce08fe41 100644 --- a/nornir/core/__init__.py +++ b/nornir/core/__init__.py @@ -131,7 +131,7 @@ def configure_logging(self): if dictConfig["root"]["handlers"]: logging.config.dictConfig(dictConfig) - def filter(self, **kwargs): + def filter(self, *args, **kwargs): """ See :py:meth:`nornir.core.inventory.Inventory.filter` @@ -139,7 +139,7 @@ def filter(self, **kwargs): :obj:`Nornir`: A new object with same configuration as ``self`` but filtered inventory. """ b = Nornir(dry_run=self.dry_run, **self.__dict__) - b.inventory = self.inventory.filter(**kwargs) + b.inventory = self.inventory.filter(*args, **kwargs) return b def _run_serial(self, task, hosts, **kwargs): diff --git a/nornir/core/filter.py b/nornir/core/filter.py new file mode 100644 index 00000000..8a67687b --- /dev/null +++ b/nornir/core/filter.py @@ -0,0 +1,88 @@ +class F_OP_BASE(object): + + def __init__(self, op1, op2): + self.op1 = op1 + self.op2 = op2 + + def __and__(self, other): + return AND(self, other) + + def __or__(self, other): + return OR(self, other) + + def __repr__(self): + return "( {} {} {} )".format(self.op1, self.__class__.__name__, self.op2) + + +class AND(F_OP_BASE): + + def __call__(self, host): + return self.op1(host) and self.op2(host) + + +class OR(F_OP_BASE): + + def __call__(self, host): + return self.op1(host) or self.op2(host) + + +class F(object): + + def __init__(self, **kwargs): + self.filters = kwargs + + def __call__(self, host): + return all( + F._verify_rules(host, k.split("__"), v) for k, v in self.filters.items() + ) + + def __and__(self, other): + return AND(self, other) + + def __or__(self, other): + return OR(self, other) + + def __invert__(self): + return NOT_F(**self.filters) + + def __repr__(self): + return "".format(self.filters) + + @staticmethod + def _verify_rules(data, rule, value): + if len(rule) > 1: + return F._verify_rules(data.get(rule[0], {}), rule[1:], value) + + elif len(rule) == 1: + operator = "__{}__".format(rule[0]) + if hasattr(data, operator): + return getattr(data, operator)(value) + + elif hasattr(data, rule[0]): + if callable(getattr(data, rule[0])): + return getattr(data, rule[0])(value) + + else: + return getattr(data, rule[0]) == value + + else: + return data.get(rule[0]) == value + + else: + raise Exception( + "I don't know how I got here:\n{}\n{}\n{}".format(data, rule, value) + ) + + +class NOT_F(F): + + def __call__(self, host): + return not any( + F._verify_rules(host, k.split("__"), v) for k, v in self.filters.items() + ) + + def __invert__(self): + return F(**self.filters) + + def __repr__(self): + return "".format(self.filters) diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 7563188b..37744888 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -113,11 +113,21 @@ def to_dict(self): def has_parent_group(self, group): """Retuns whether the object is a child of the :obj:`Group` ``group``""" + if isinstance(group, str): + return self._has_parent_group_by_name(group) + + else: + return self._has_parent_group_by_object(group) + + def _has_parent_group_by_name(self, group): for g in self.groups: - if g is group or g.has_parent_group(group): + if g.name == group or g.has_parent_group(group): return True - return False + def _has_parent_group_by_object(self, group): + for g in self.groups: + if g is group or g.has_parent_group(group): + return True def __getitem__(self, item): try: @@ -317,7 +327,7 @@ def _resolve_groups(self, groups): r = groups return r - def filter(self, filter_func=None, **kwargs): + def filter(self, filter_obj=None, filter_func=None, *args, **kwargs): """ Returns a new inventory after filtering the hosts by matching the data passed to the function. For instance, assume an inventory with:: @@ -337,9 +347,11 @@ def filter(self, filter_func=None, **kwargs): * ``my_inventory.filter(site="bma", role="db")`` will result in ``host3`` only Arguments: + filter_obj (:obj:nornir.core.filter.F): Filter object to run filter_func (callable): if filter_func is passed it will be called against each device. If the call returns ``True`` the device will be kept in the inventory """ + filter_func = filter_obj or filter_func if filter_func: filtered = {n: h for n, h in self.hosts.items() if filter_func(h, **kwargs)} else: diff --git a/tests/core/test_filter.py b/tests/core/test_filter.py new file mode 100644 index 00000000..a8c3018d --- /dev/null +++ b/tests/core/test_filter.py @@ -0,0 +1,114 @@ +import os + +from nornir.core.filter import F +from nornir.plugins.inventory.simple import SimpleInventory + +dir_path = os.path.dirname(os.path.realpath(__file__)) +inventory = SimpleInventory( + "{}/../inventory_data/hosts.yaml".format(dir_path), + "{}/../inventory_data/groups.yaml".format(dir_path), +) + + +class Test(object): + + def test_simple(self): + f = F(site="site1") + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev1.group_1", "dev2.group_1"] + + def test_and(self): + f = F(site="site1") & F(role="www") + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev1.group_1"] + + def test_or(self): + f = F(site="site1") | F(role="www") + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev1.group_1", "dev2.group_1", "dev3.group_2"] + + def test_combined(self): + f = F(site="site2") | (F(role="www") & F(my_var="comes_from_dev1.group_1")) + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev1.group_1", "dev3.group_2", "dev4.group_2"] + + f = (F(site="site2") | F(role="www")) & F(my_var="comes_from_dev1.group_1") + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev1.group_1"] + + def test_contains(self): + f = F(groups__contains="group_1") + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev1.group_1", "dev2.group_1"] + + def test_negate(self): + f = ~F(groups__contains="group_1") + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev3.group_2", "dev4.group_2"] + + def test_negate_and_second_negate(self): + f = F(site="site1") & ~F(role="www") + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev2.group_1"] + + def test_negate_or_both_negate(self): + f = ~F(site="site1") | ~F(role="www") + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev2.group_1", "dev3.group_2", "dev4.group_2"] + + def test_nested_data_a_string(self): + f = F(nested_data__a_string="asdasd") + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev1.group_1"] + + def test_nested_data_a_string_contains(self): + f = F(nested_data__a_string__contains="asd") + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev1.group_1"] + + def test_nested_data_a_dict_contains(self): + f = F(nested_data__a_dict__contains="a") + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev1.group_1"] + + def test_nested_data_a_dict_element(self): + f = F(nested_data__a_dict__a=1) + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev1.group_1"] + + def test_nested_data_a_dict_doesnt_contain(self): + f = ~F(nested_data__a_dict__contains="a") + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev2.group_1", "dev3.group_2", "dev4.group_2"] + + def test_nested_data_a_list_contains(self): + f = F(nested_data__a_list__contains=2) + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev1.group_1", "dev2.group_1"] + + def test_filtering_by_callable_has_parent_group(self): + f = F(has_parent_group="parent_group") + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev1.group_1", "dev2.group_1"] + + def test_filtering_by_attribute_name(self): + f = F(name="dev1.group_1") + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev1.group_1"] diff --git a/tests/core/test_inventory.py b/tests/core/test_inventory.py index 055b4a7e..600d8cfe 100644 --- a/tests/core/test_inventory.py +++ b/tests/core/test_inventory.py @@ -204,12 +204,19 @@ def test_has_parents(self): assert not inventory.hosts["dev1.group_1"].has_parent_group( inventory.groups["group_2"] ) + assert inventory.hosts["dev1.group_1"].has_parent_group("group_1") + assert not inventory.hosts["dev1.group_1"].has_parent_group("group_2") def test_to_dict(self): expected = { "hosts": { "dev1.group_1": { "name": "dev1.group_1", + "nested_data": { + "a_dict": {"a": 1, "b": 2}, + "a_list": [1, 2], + "a_string": "asdasd", + }, "groups": ["group_1"], "my_var": "comes_from_dev1.group_1", "www_server": "nginx", @@ -230,8 +237,12 @@ def test_to_dict(self): }, "groups": { "defaults": {}, + "parent_group": {"a_var": "blah", "name": "parent_group"}, "group_1": { - "name": "group_1", "my_var": "comes_from_group_1", "site": "site1" + "name": "group_1", + "my_var": "comes_from_group_1", + "site": "site1", + "groups": ["parent_group"], }, "group_2": {"name": "group_2", "site": "site2"}, }, diff --git a/tests/inventory_data/groups.yaml b/tests/inventory_data/groups.yaml index 81ecaa51..fc8b14ae 100644 --- a/tests/inventory_data/groups.yaml +++ b/tests/inventory_data/groups.yaml @@ -6,9 +6,14 @@ defaults: nornir_password: docker nornir_os: linux +parent_group: + a_var: blah + group_1: my_var: comes_from_group_1 site: site1 + groups: + - parent_group group_2: site: site2 diff --git a/tests/inventory_data/hosts.yaml b/tests/inventory_data/hosts.yaml index 86078f08..329d4b57 100644 --- a/tests/inventory_data/hosts.yaml +++ b/tests/inventory_data/hosts.yaml @@ -7,6 +7,12 @@ dev1.group_1: role: www nornir_ssh_port: 65001 nornir_nos: eos + nested_data: + a_dict: + a: 1 + b: 2 + a_list: [1, 2] + a_string: "asdasd" dev2.group_1: groups: @@ -14,6 +20,12 @@ dev2.group_1: role: db nornir_ssh_port: 65002 nornir_nos: junos + nested_data: + a_dict: + b: 2 + c: 3 + a_list: [2, 3] + a_string: "qwe" dev3.group_2: groups: diff --git a/tests/plugins/inventory/helpers/__init__.py b/tests/plugins/inventory/helpers/__init__.py new file mode 100644 index 00000000..e69de29b From 6de87eec07fed3cd454da17b440a301ebe615470 Mon Sep 17 00:00:00 2001 From: Dmitry Figol Date: Wed, 18 Jul 2018 16:30:47 +0300 Subject: [PATCH 005/109] Add annotations to nornir.plugins and mypy to CI (#187) --- .gitignore | 1 + .travis.yml | 2 + nornir/core/inventory.py | 6 + nornir/core/task.py | 6 +- nornir/plugins/functions/text/__init__.py | 60 ++++--- nornir/plugins/inventory/ansible.py | 166 ++++++++++++------ nornir/plugins/inventory/nsot.py | 51 +++--- nornir/plugins/inventory/simple.py | 17 +- nornir/plugins/tasks/apis/http_method.py | 21 ++- nornir/plugins/tasks/commands/command.py | 11 +- .../plugins/tasks/commands/remote_command.py | 10 +- .../tasks/connections/napalm_connection.py | 16 +- .../tasks/connections/netmiko_connection.py | 6 +- .../tasks/connections/paramiko_connection.py | 4 +- nornir/plugins/tasks/data/load_json.py | 15 +- nornir/plugins/tasks/data/load_yaml.py | 15 +- nornir/plugins/tasks/files/sftp.py | 54 ++++-- nornir/plugins/tasks/files/write_file.py | 34 ++-- nornir/plugins/tasks/networking/napalm_cli.py | 12 +- .../tasks/networking/napalm_configure.py | 24 ++- nornir/plugins/tasks/networking/napalm_get.py | 16 +- .../tasks/networking/napalm_validate.py | 15 +- .../tasks/networking/netmiko_file_transfer.py | 15 +- .../tasks/networking/netmiko_send_command.py | 16 +- .../tasks/networking/netmiko_send_config.py | 23 ++- nornir/plugins/tasks/networking/tcp_ping.py | 14 +- nornir/plugins/tasks/text/template_file.py | 24 ++- nornir/plugins/tasks/text/template_string.py | 14 +- requirements-dev.txt | 1 + requirements.txt | 1 + setup.cfg | 20 +++ tests/plugins/tasks/data/test_load_yaml.py | 1 - tox.ini | 8 + 33 files changed, 460 insertions(+), 239 deletions(-) diff --git a/.gitignore b/.gitignore index 9ada0655..36cb2695 100644 --- a/.gitignore +++ b/.gitignore @@ -101,5 +101,6 @@ output/ .DS_Store .pytest_cache/ +.mypy_cache/ .vscode diff --git a/.travis.yml b/.travis.yml index 392619a1..0c930391 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,8 @@ matrix: env: TOXENV=black - python: 3.6 env: TOXENV=pylama + - python: 3.6 + env: TOXENV=mypy install: - pip install tox tox-travis coveralls script: diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 37744888..eda727ae 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -1,5 +1,11 @@ +from typing import Dict, Any + import getpass +VarsDict = Dict[str, Any] +HostsDict = Dict[str, VarsDict] +GroupsDict = Dict[str, VarsDict] + class Host(object): """ diff --git a/nornir/core/task.py b/nornir/core/task.py index 74e49c9e..c26afbe0 100644 --- a/nornir/core/task.py +++ b/nornir/core/task.py @@ -1,6 +1,7 @@ import logging import traceback -from builtins import super + +from typing import Optional from nornir.core.exceptions import NornirExecutionError from nornir.core.exceptions import NornirSubTaskError @@ -163,6 +164,9 @@ def __init__( self.name = None self.severity_level = severity_level + self.stdout: Optional[str] = None + self.stderr: Optional[str] = None + for k, v in kwargs.items(): setattr(self, k, v) diff --git a/nornir/plugins/functions/text/__init__.py b/nornir/plugins/functions/text/__init__.py index fb1f8c99..10418033 100644 --- a/nornir/plugins/functions/text/__init__.py +++ b/nornir/plugins/functions/text/__init__.py @@ -1,6 +1,7 @@ import logging import pprint import threading +from typing import List, Optional from colorama import Fore, Style, init @@ -13,7 +14,7 @@ init(autoreset=True, convert=False, strip=False) -def print_title(title): +def print_title(title: str) -> None: """ Helper function to print a title. """ @@ -21,7 +22,7 @@ def print_title(title): print("{}{}{}{}".format(Style.BRIGHT, Fore.GREEN, msg, "*" * (80 - len(msg)))) -def _get_color(result, failed): +def _get_color(result: Result, failed: bool) -> str: if result.failed or failed: color = Fore.RED elif result.changed: @@ -32,8 +33,13 @@ def _get_color(result, failed): def _print_individual_result( - result, host, vars, failed, severity_level, task_group=False -): + result: Result, + host: Optional[str], + attrs: List[str], + failed: bool, + severity_level: int, + task_group: bool = False, +) -> None: if result.severity_level < severity_level: return @@ -49,8 +55,8 @@ def _print_individual_result( Style.BRIGHT, color, msg, symbol * (80 - len(msg)), level_name ) ) - for v in vars: - x = getattr(result, v, "") + for attribute in attrs: + x = getattr(result, attribute, "") if x and not isinstance(x, str): pprint.pprint(x, indent=2) elif x: @@ -58,11 +64,15 @@ def _print_individual_result( def _print_result( - result, host=None, vars=None, failed=None, severity_level=logging.INFO -): - vars = vars or ["diff", "result", "stdout"] - if isinstance(vars, str): - vars = [vars] + result: Result, + host: Optional[str] = None, + attrs: List[str] = None, + failed: bool = False, + severity_level: int = logging.INFO, +) -> None: + attrs = attrs or ["diff", "result", "stdout"] + if isinstance(attrs, str): + attrs = [attrs] if isinstance(result, AggregatedResult): msg = result.name @@ -75,34 +85,36 @@ def _print_result( print( "{}{}{}{}".format(Style.BRIGHT, Fore.BLUE, msg, "*" * (80 - len(msg))) ) - _print_result(host_data, host, vars, failed, severity_level) + _print_result(host_data, host, attrs, failed, severity_level) elif isinstance(result, MultiResult): _print_individual_result( - result[0], host, vars, failed, severity_level, task_group=True + result[0], host, attrs, failed, severity_level, task_group=True ) for r in result[1:]: - _print_result(r, host, vars, failed, severity_level) + _print_result(r, host, attrs, failed, severity_level) color = _get_color(result[0], failed) msg = "^^^^ END {} ".format(result[0].name) print("{}{}{}{}".format(Style.BRIGHT, color, msg, "^" * (80 - len(msg)))) elif isinstance(result, Result): - _print_individual_result(result, host, vars, failed, severity_level) + _print_individual_result(result, host, attrs, failed, severity_level) def print_result( - result, host=None, vars=None, failed=None, severity_level=logging.INFO -): + result: Result, + host: Optional[str] = None, + vars: List[str] = None, + failed: bool = False, + severity_level: int = logging.INFO, +) -> None: """ Prints the :obj:`nornir.core.task.Result` from a previous task to screen Arguments: - result (:obj:`nornir.core.task.Result`): from a previous task - vars (list of str): Which attributes you want to print - failed (``bool``): if ``True`` assume the task failed - severity_level (int): Print only errors with this severity level or higher - - Returns: - :obj:`nornir.core.task.Result`: + result: from a previous task + host: # TODO + vars: Which attributes you want to print + failed: if ``True`` assume the task failed + severity_level: Print only errors with this severity level or higher """ LOCK.acquire() try: diff --git a/nornir/plugins/inventory/ansible.py b/nornir/plugins/inventory/ansible.py index cc16ba17..680c4ed0 100644 --- a/nornir/plugins/inventory/ansible.py +++ b/nornir/plugins/inventory/ansible.py @@ -1,30 +1,48 @@ try: import configparser as cp except ImportError: - import ConfigParser as cp + # https://github.com/python/mypy/issues/1153 + import ConfigParser as cp # type: ignore import logging import os -from builtins import super - -from nornir.core.inventory import Inventory +from collections import defaultdict +from typing import Dict, Any, Tuple, Optional, cast, Union, MutableMapping, DefaultDict +from mypy_extensions import TypedDict import ruamel.yaml +from ruamel.yaml.scanner import ScannerError +from ruamel.yaml.composer import ComposerError + +from nornir.core.inventory import Inventory, VarsDict, GroupsDict, HostsDict logger = logging.getLogger("nornir") +AnsibleHostsDict = Dict[str, Optional[VarsDict]] + +AnsibleGroupDataDict = TypedDict( + "AnsibleGroupDataDict", + {"children": Dict[str, Any], "vars": VarsDict, "hosts": AnsibleHostsDict}, + total=False, +) # bug: https://github.com/python/mypy/issues/5357 + +AnsibleGroupsDict = Dict[str, AnsibleGroupDataDict] + + class AnsibleParser(object): - def __init__(self, hostsfile): + def __init__(self, hostsfile: str) -> None: self.hostsfile = hostsfile self.path = os.path.dirname(hostsfile) - self.hosts = {} - self.groups = {} - self.original_data = None + self.hosts: HostsDict = {} + self.groups: GroupsDict = {} + self.original_data: Optional[AnsibleGroupsDict] = None self.load_hosts_file() - def parse_group(self, group, data, parent=None): + def parse_group( + self, group: str, data: AnsibleGroupDataDict, parent: Optional[str] = None + ) -> None: data = data or {} if group == "defaults": self.groups[group] = {} @@ -43,13 +61,18 @@ def parse_group(self, group, data, parent=None): self.parse_hosts(data.get("hosts", {}), parent=group) for children, children_data in data.get("children", {}).items(): - self.parse_group(children, children_data, parent=group) + self.parse_group( + children, cast(AnsibleGroupDataDict, children_data), parent=group + ) - def parse(self): - self.parse_group("defaults", self.original_data["all"]) + def parse(self) -> None: + if self.original_data is not None: + self.parse_group("defaults", self.original_data["all"]) self.sort_groups() - def parse_hosts(self, hosts, parent=None): + def parse_hosts( + self, hosts: AnsibleHostsDict, parent: Optional[str] = None + ) -> None: for host, data in hosts.items(): data = data or {} self.add(host, self.hosts) @@ -59,7 +82,7 @@ def parse_hosts(self, hosts, parent=None): self.hosts[host].update(self.read_vars_file(host, self.path, True)) self.hosts[host] = self.map_nornir_vars(self.hosts[host]) - def sort_groups(self): + def sort_groups(self) -> None: for host in self.hosts.values(): host["groups"].sort() @@ -69,7 +92,8 @@ def sort_groups(self): group["groups"].sort() - def read_vars_file(self, element, path, is_host=True): + @staticmethod + def read_vars_file(element: str, path: str, is_host: bool = True) -> VarsDict: subdir = "host_vars" if is_host else "group_vars" filepath = os.path.join(path, subdir, element) @@ -84,7 +108,8 @@ def read_vars_file(self, element, path, is_host=True): yml = ruamel.yaml.YAML(typ="rt", pure=True) return yml.load(f) - def map_nornir_vars(self, obj): + @staticmethod + def map_nornir_vars(obj: VarsDict): mappings = { "ansible_host": "nornir_host", "ansible_port": "nornir_ssh_port", @@ -99,84 +124,109 @@ def map_nornir_vars(self, obj): result[k] = v return result - def add(self, element, element_dict): + @staticmethod + def add(element: str, element_dict: Dict[str, VarsDict]) -> None: if element not in element_dict: element_dict[element] = {"groups": []} + def load_hosts_file(self) -> None: + raise NotImplementedError + class INIParser(AnsibleParser): - def normalize_content(self, content): - result = {} + @staticmethod + def normalize_value(value: str) -> Union[str, int]: + try: + return int(value) + + except (ValueError, TypeError): + return value + + @staticmethod + def normalize_content(content: str) -> VarsDict: + result: VarsDict = {} if not content: return result for option in content.split(): - k, v = option.split("=") - try: - v = int(v) - except Exception: - pass - result[k] = v - + key, value = option.split("=") + result[key] = INIParser.normalize_value(value) return result - def normalize(self, data): - result = {} - for k, v in data.items(): - meta = None - if ":" in k: - k, meta = k.split(":") - if k not in result: - result[k] = {} - - if meta not in result[k]: - result[k][meta] = {} - - if meta == "vars": - for data, _ in v.items(): - result[k][meta].update(self.normalize_content(data)) - elif meta == "children": - result[k][meta] = {k: {} for k, _ in v.items()} + @staticmethod + def process_meta( + meta: Optional[str], section: MutableMapping[str, str] + ) -> Dict[str, Any]: + if meta == "vars": + return { + key: INIParser.normalize_value(value) for key, value in section.items() + } + + elif meta == "children": + return {group_name: {} for group_name in section} + + else: + raise ValueError(f"Unknown tag {meta}") + + def normalize(self, data: cp.ConfigParser) -> Dict[str, AnsibleGroupDataDict]: + groups: DefaultDict[str, Dict[str, Any]] = defaultdict(dict) + # Dict[str, AnsibleGroupDataDict] does not work because of + # https://github.com/python/mypy/issues/5359 + result: Dict[str, Dict[str, Dict[str, Dict[str, Any]]]] = { + "all": {"children": groups} + } + + for section_name, section in data.items(): + + if section_name == "DEFAULT": + continue + + if ":" in section_name: + group_name, meta = section_name.split(":") + subsection = self.process_meta(meta, section) + if group_name == "all": + result["all"][meta] = subsection + else: + groups[group_name][meta] = subsection else: - result[k]["hosts"] = { - host: self.normalize_content(data) for host, data in v.items() + groups[section_name]["hosts"] = { + host: self.normalize_content(host_vars) + for host, host_vars in section.items() } - return result + return cast(AnsibleGroupsDict, result) - def load_hosts_file(self): + def load_hosts_file(self) -> None: original_data = cp.ConfigParser( - interpolation=None, allow_no_value=True, delimiters=" " + interpolation=None, allow_no_value=True, delimiters=" =" ) original_data.read(self.hostsfile) - data = self.normalize(original_data) - data.pop("DEFAULT") - if "all" not in data: - self.original_data = {"all": {"children": data}} + self.original_data = self.normalize(original_data) class YAMLParser(AnsibleParser): - def load_hosts_file(self): + def load_hosts_file(self) -> None: with open(self.hostsfile, "r") as f: yml = ruamel.yaml.YAML(typ="rt", pure=True) - self.original_data = yml.load(f.read()) + self.original_data = cast(AnsibleGroupsDict, yml.load(f.read())) -def parse(hostsfile): +def parse(hostsfile: str) -> Tuple[HostsDict, GroupsDict]: try: - parser = INIParser(hostsfile) + parser: AnsibleParser = INIParser(hostsfile) except cp.Error: try: parser = YAMLParser(hostsfile) - except ruamel.yaml.scanner.ScannerError: + except (ScannerError, ComposerError): logger.error( "couldn't parse '{}' as neither a ini nor yaml file".format(hostsfile) ) raise parser.parse() + return parser.hosts, parser.groups @@ -188,7 +238,7 @@ class AnsibleInventory(Inventory): hostsfile (string): Ansible inventory file to load """ - def __init__(self, hostsfile="hosts", **kwargs): + def __init__(self, hostsfile: str = "hosts", **kwargs: Any) -> None: host_vars, group_vars = parse(hostsfile) defaults = group_vars.pop("defaults") super().__init__(host_vars, group_vars, defaults, **kwargs) diff --git a/nornir/plugins/inventory/nsot.py b/nornir/plugins/inventory/nsot.py index 569c7e31..6fff8913 100644 --- a/nornir/plugins/inventory/nsot.py +++ b/nornir/plugins/inventory/nsot.py @@ -1,10 +1,10 @@ import os -from builtins import super - -from nornir.core.inventory import Inventory +from typing import Any, List import requests +from nornir.core.inventory import Inventory, VarsDict, HostsDict + class NSOTInventory(Inventory): """ @@ -21,31 +21,30 @@ class NSOTInventory(Inventory): * ``NSOT_SECRET_KEY``: Corresponds to nsot_secret_key argument Arguments: - flatten_attributes (bool): Assign host attributes to the root object. Useful + flatten_attributes: Assign host attributes to the root object. Useful for filtering hosts. - nsot_url (string): URL to nsot's API (defaults to ``http://localhost:8990/api``) - nsot_email (string): email for authtication (defaults to admin@acme.com) - nsot_auth_header (string): String for auth_header authentication (defaults to X-NSoT-Email) - nsot_secret_key (string): Secret Key for auth_token method. If given auth_token + nsot_url: URL to nsot's API (defaults to ``http://localhost:8990/api``) + nsot_email: email for authtication (defaults to admin@acme.com) + nsot_auth_header: String for auth_header authentication (defaults to X-NSoT-Email) + nsot_secret_key: Secret Key for auth_token method. If given auth_token will be used as auth_method. """ def __init__( self, - nsot_url="", - nsot_email="", - nsot_auth_method="", - nsot_secret_key="", - nsot_auth_header="", - flatten_attributes=True, - **kwargs - ): + nsot_url: str = "", + nsot_email: str = "", + nsot_secret_key: str = "", + nsot_auth_header: str = "", + flatten_attributes: bool = True, + **kwargs: Any + ) -> None: nsot_url = nsot_url or os.environ.get("NSOT_URL", "http://localhost:8990/api") nsot_email = nsot_email or os.environ.get("NSOT_EMAIL", "admin@acme.com") - nsot_secret_key = nsot_secret_key or os.environ.get("NSOT_SECRET_KEY") + secret_key = nsot_secret_key or os.environ.get("NSOT_SECRET_KEY") - if nsot_secret_key: - data = {"email": nsot_email, "secret_key": nsot_secret_key} + if secret_key: + data = {"email": nsot_email, "secret_key": secret_key} res = requests.post("{}/authenticate/".format(nsot_url), data=data) auth_token = res.json().get("auth_token") headers = { @@ -58,9 +57,13 @@ def __init__( ) headers = {nsot_auth_header: nsot_email} - devices = requests.get("{}/devices".format(nsot_url), headers=headers).json() - sites = requests.get("{}/sites".format(nsot_url), headers=headers).json() - interfaces = requests.get( + devices: List[VarsDict] = requests.get( + "{}/devices".format(nsot_url), headers=headers + ).json() + sites: List[VarsDict] = requests.get( + "{}/sites".format(nsot_url), headers=headers + ).json() + interfaces: List[VarsDict] = requests.get( "{}/interfaces".format(nsot_url), headers=headers ).json() @@ -79,6 +82,6 @@ def __init__( devices[i["device"] - 1]["interfaces"][i["name"]] = i # Finally the inventory expects a dict of hosts where the key is the hostname - devices = {d["hostname"]: d for d in devices} + hosts: HostsDict = {d["hostname"]: d for d in devices} - super().__init__(devices, None, **kwargs) + super().__init__(hosts, None, **kwargs) diff --git a/nornir/plugins/inventory/simple.py b/nornir/plugins/inventory/simple.py index 03b87039..9874a5d7 100644 --- a/nornir/plugins/inventory/simple.py +++ b/nornir/plugins/inventory/simple.py @@ -1,8 +1,8 @@ import logging import os -from builtins import super +from typing import Any -from nornir.core.inventory import Inventory +from nornir.core.inventory import Inventory, GroupsDict, HostsDict, VarsDict import ruamel.yaml @@ -116,19 +116,24 @@ class SimpleInventory(Inventory): domain: cmh.acme.com """ - def __init__(self, host_file="hosts.yaml", group_file="groups.yaml", **kwargs): + def __init__( + self, + host_file: str = "hosts.yaml", + group_file: str = "groups.yaml", + **kwargs: Any + ) -> None: with open(host_file, "r") as f: - hosts = ruamel.yaml.safe_load(f.read()) + hosts: HostsDict = ruamel.yaml.safe_load(f.read()) if group_file: if os.path.exists(group_file): with open(group_file, "r") as f: - groups = ruamel.yaml.safe_load(f.read()) + groups: GroupsDict = ruamel.yaml.safe_load(f.read()) else: logging.warning("{}: doesn't exist".format(group_file)) groups = {} else: groups = {} - defaults = groups.pop("defaults", {}) + defaults: VarsDict = groups.pop("defaults", {}) super().__init__(hosts, groups, defaults, **kwargs) diff --git a/nornir/plugins/tasks/apis/http_method.py b/nornir/plugins/tasks/apis/http_method.py index e4c50292..6823184f 100644 --- a/nornir/plugins/tasks/apis/http_method.py +++ b/nornir/plugins/tasks/apis/http_method.py @@ -1,17 +1,24 @@ -from nornir.core.task import Result +from typing import Optional, Any +from nornir.core.task import Result, Task import requests -def http_method(task=None, method="get", url="", raise_for_status=True, **kwargs): +def http_method( + task: Optional[Task] = None, + method: str = "get", + url: str = "", + raise_for_status: bool = True, + **kwargs: Any +) -> Result: """ This is a convenience task that uses `requests `_ to interact with an HTTP server. Arguments: - method (string): HTTP method to call - url (string): URL to connect to - raise_for_status (bool): Whether to call `raise_for_status + method: HTTP method to call + url: URL to connect to + raise_for_status: Whether to call `raise_for_status `_ method automatically or not. For quick reference, raise_for_status will consider an error if the return code is any of 4xx or 5xx @@ -20,10 +27,10 @@ def http_method(task=None, method="get", url="", raise_for_status=True, **kwargs method Returns: - :obj:`nornir.core.task.Result`: + Result object with the following attributes set: * result (``string/dict``): Body of the response. Either text or a dict if the response was a json object - * reponse (object): Original `Response + * response (``requests.Response``): Original `Response `_ """ r = requests.request(method, url, **kwargs) diff --git a/nornir/plugins/tasks/commands/command.py b/nornir/plugins/tasks/commands/command.py index f97ace74..154e9deb 100644 --- a/nornir/plugins/tasks/commands/command.py +++ b/nornir/plugins/tasks/commands/command.py @@ -3,21 +3,22 @@ from nornir.core.exceptions import CommandError -from nornir.core.task import Result +from nornir.core.task import Result, Task -def command(task, command): +def command(task: Task, command: str) -> Result: """ Executes a command locally Arguments: - command (``str``): command to execute + command: command to execute Returns: - :obj:`nornir.core.task.Result`: + Result object with the following attributes set: * result (``str``): stderr or stdout * stdout (``str``): stdout - * stderr (``srr``): stderr + * stderr (``str``): stderr + Raises: :obj:`nornir.core.exceptions.CommandError`: when there is a command error """ diff --git a/nornir/plugins/tasks/commands/remote_command.py b/nornir/plugins/tasks/commands/remote_command.py index d4b5567d..d6ee81ec 100644 --- a/nornir/plugins/tasks/commands/remote_command.py +++ b/nornir/plugins/tasks/commands/remote_command.py @@ -1,21 +1,21 @@ from nornir.core.exceptions import CommandError -from nornir.core.task import Result +from nornir.core.task import Result, Task from paramiko.agent import AgentRequestHandler -def remote_command(task, command): +def remote_command(task: Task, command: str) -> Result: """ - Executes a command locally + Executes a command remotely on the host Arguments: command (``str``): command to execute Returns: - :obj:`nornir.core.task.Result`: + Result object with the following attributes set: * result (``str``): stderr or stdout * stdout (``str``): stdout - * stderr (``srr``): stderr + * stderr (``str``): stderr Raises: :obj:`nornir.core.exceptions.CommandError`: when there is a command error diff --git a/nornir/plugins/tasks/connections/napalm_connection.py b/nornir/plugins/tasks/connections/napalm_connection.py index fb4f8d4d..77bf0651 100644 --- a/nornir/plugins/tasks/connections/napalm_connection.py +++ b/nornir/plugins/tasks/connections/napalm_connection.py @@ -1,14 +1,20 @@ +from typing import Optional, Dict, Any + from napalm import get_network_driver +from nornir.core.task import Task + -def napalm_connection(task=None, timeout=60, optional_args=None): +def napalm_connection( + task: Task, timeout: int = 60, optional_args: Optional[Dict[str, Any]] = None +) -> None: """ - This tasks connects to the device using the NAPALM driver and sets the + This task connects to the device using the NAPALM driver and sets the relevant connection. Arguments: - timeout (int, optional): defaults to 60 - optional_args (dict, optional): defaults to `{"port": task.host["nornir_network_api_port"]}` + timeout: defaults to 60 + optional_args: defaults to `{"port": task.host["nornir_network_api_port"]}` Inventory: napalm_options: maps directly to ``optional_args`` when establishing the connection @@ -39,7 +45,7 @@ def napalm_connection(task=None, timeout=60, optional_args=None): if platform == "nxos": if not host.network_api_port: if host.ssh_port or parameters["optional_args"].get("port") == 22: - platform == "nxos_ssh" + platform = "nxos_ssh" # Fallback for community drivers (priority api_port over ssh_port) if platform not in (api_platforms + ssh_platforms): diff --git a/nornir/plugins/tasks/connections/netmiko_connection.py b/nornir/plugins/tasks/connections/netmiko_connection.py index af33d8e3..5664eb52 100644 --- a/nornir/plugins/tasks/connections/netmiko_connection.py +++ b/nornir/plugins/tasks/connections/netmiko_connection.py @@ -1,5 +1,9 @@ +from typing import Any + from netmiko import ConnectHandler +from nornir.core.task import Task + napalm_to_netmiko_map = { "ios": "cisco_ios", "nxos": "cisco_nxos", @@ -9,7 +13,7 @@ } -def netmiko_connection(task, **netmiko_args): +def netmiko_connection(task: Task, **netmiko_args: Any) -> None: """Connect to the host using Netmiko and set the relevant connection in the connection map. Precedence: ``**netmiko_args`` > discrete inventory attributes > inventory netmiko_options diff --git a/nornir/plugins/tasks/connections/paramiko_connection.py b/nornir/plugins/tasks/connections/paramiko_connection.py index c494c720..ecdb0a89 100644 --- a/nornir/plugins/tasks/connections/paramiko_connection.py +++ b/nornir/plugins/tasks/connections/paramiko_connection.py @@ -2,8 +2,10 @@ import paramiko +from nornir.core.task import Task -def paramiko_connection(task=None): + +def paramiko_connection(task: Task) -> None: """ This tasks connects to the device with paramiko to the device and sets the relevant connection. diff --git a/nornir/plugins/tasks/data/load_json.py b/nornir/plugins/tasks/data/load_json.py index 453e9f6f..269a1da8 100644 --- a/nornir/plugins/tasks/data/load_json.py +++ b/nornir/plugins/tasks/data/load_json.py @@ -1,16 +1,17 @@ import collections import json +from typing import Dict, MutableMapping, Any, Type -from nornir.core.task import Result +from nornir.core.task import Result, Task -def load_json(task, file, ordered_dict=False): +def load_json(task: Task, file: str, ordered_dict: bool = False) -> Result: """ Loads a json file. Arguments: - file (str): path to the file containing the json file to load - ordered_dict (bool): If set to true used OrderedDict to load maps + file: path to the file containing the json file to load + ordered_dict: If set to true used OrderedDict to load maps Examples: @@ -20,11 +21,13 @@ def load_json(task, file, ordered_dict=False): file="mydata.json", ordered_dict=True) + file: path to the file containing the json file to load + Returns: - :obj:`nornir.core.task.Result`: + Result object with the following attributes set: * result (``dict``): dictionary with the contents of the file """ - kwargs = {} + kwargs: Dict[str, Type[MutableMapping[str, Any]]] = {} if ordered_dict: kwargs["object_pairs_hook"] = collections.OrderedDict with open(file, "r") as f: diff --git a/nornir/plugins/tasks/data/load_yaml.py b/nornir/plugins/tasks/data/load_yaml.py index 52502419..327ade3d 100644 --- a/nornir/plugins/tasks/data/load_yaml.py +++ b/nornir/plugins/tasks/data/load_yaml.py @@ -1,15 +1,16 @@ -from nornir.core.task import Result - import ruamel.yaml +from typing import Dict + +from nornir.core.task import Result, Task -def load_yaml(task, file, ordered_dict=False): +def load_yaml(task: Task, file: str, ordered_dict: bool = False): """ Loads a yaml file. Arguments: - file (str): path to the file containing the yaml file to load - ordered_dict (bool): If set to true used OrderedDict to load maps + file: path to the file containing the yaml file to load + ordered_dict: If set to true used OrderedDict to load maps Examples: @@ -20,10 +21,10 @@ def load_yaml(task, file, ordered_dict=False): ordered_dict=True) Returns: - :obj:`nornir.core.task.Result`: + Result object with the following attributes set: * result (``dict``): dictionary with the contents of the file """ - kwargs = {} + kwargs: Dict[str, str] = {} kwargs["typ"] = "rt" if ordered_dict else "safe" with open(file, "r") as f: yml = ruamel.yaml.YAML(pure=True, **kwargs) diff --git a/nornir/plugins/tasks/files/sftp.py b/nornir/plugins/tasks/files/sftp.py index 3eb97435..2e85308e 100644 --- a/nornir/plugins/tasks/files/sftp.py +++ b/nornir/plugins/tasks/files/sftp.py @@ -1,9 +1,10 @@ import hashlib import os import stat +from typing import List, Optional from nornir.core.exceptions import CommandError -from nornir.core.task import Result +from nornir.core.task import Result, Task from nornir.plugins.tasks import commands import paramiko @@ -11,7 +12,7 @@ from scp import SCPClient -def get_src_hash(filename): +def get_src_hash(filename: str) -> str: sha1sum = hashlib.sha1() with open(filename, "rb") as f: @@ -22,11 +23,12 @@ def get_src_hash(filename): return sha1sum.hexdigest() -def get_dst_hash(task, filename): +def get_dst_hash(task: Task, filename: str) -> str: command = "sha1sum {}".format(filename) try: result = commands.remote_command(task, command) - return result.stdout.split()[0] + if result.stdout is not None: + return result.stdout.split()[0] except CommandError as e: if "No such file or directory" in e.stderr: @@ -34,8 +36,10 @@ def get_dst_hash(task, filename): raise + return "" -def remote_exists(sftp_client, f): + +def remote_exists(sftp_client: paramiko.SFTPClient, f: str) -> bool: try: sftp_client.stat(f) return True @@ -44,7 +48,9 @@ def remote_exists(sftp_client, f): return False -def compare_put_files(task, sftp_client, src, dst): +def compare_put_files( + task: Task, sftp_client: paramiko.SFTPClient, src: str, dst: str +) -> List[str]: changed = [] if os.path.isfile(src): src_hash = get_src_hash(src) @@ -65,7 +71,9 @@ def compare_put_files(task, sftp_client, src, dst): return changed -def compare_get_files(task, sftp_client, src, dst): +def compare_get_files( + task: Task, sftp_client: paramiko.SFTPClient, src: str, dst: str +) -> List[str]: changed = [] if stat.S_ISREG(sftp_client.stat(src).st_mode): # is a file @@ -87,21 +95,37 @@ def compare_get_files(task, sftp_client, src, dst): return changed -def get(task, scp_client, sftp_client, src, dst, dry_run, *args, **kwargs): +def get( + task: Task, + scp_client: SCPClient, + sftp_client: paramiko.SFTPClient, + src: str, + dst: str, + dry_run: Optional[bool] = None, +) -> List[str]: changed = compare_get_files(task, sftp_client, src, dst) if changed and not dry_run: scp_client.get(src, dst, recursive=True) return changed -def put(task, scp_client, sftp_client, src, dst, dry_run, *args, **kwargs): +def put( + task: Task, + scp_client: SCPClient, + sftp_client: paramiko.SFTPClient, + src: str, + dst: str, + dry_run: Optional[bool] = None, +) -> List[str]: changed = compare_put_files(task, sftp_client, src, dst) if changed and not dry_run: scp_client.put(src, dst, recursive=True) return changed -def sftp(task, src, dst, action, dry_run=None): +def sftp( + task: Task, src: str, dst: str, action: str, dry_run: Optional[bool] = None +) -> Result: """ Transfer files from/to the device using sftp protocol @@ -113,13 +137,13 @@ def sftp(task, src, dst, action, dry_run=None): dst="/tmp/README.md") Arguments: - dry_run (bool): Whether to apply changes or not - src (``str``): source file - dst (``str``): destination - action (``str``): ``put``, ``get``. + dry_run: Whether to apply changes or not + src: source file + dst: destination + action: ``put``, ``get``. Returns: - :obj:`nornir.core.task.Result`: + Result object with the following attributes set: * changed (``bool``): * files_changed (``list``): list of files that changed """ diff --git a/nornir/plugins/tasks/files/write_file.py b/nornir/plugins/tasks/files/write_file.py index a1f2a6b2..23b1b7b3 100644 --- a/nornir/plugins/tasks/files/write_file.py +++ b/nornir/plugins/tasks/files/write_file.py @@ -1,10 +1,11 @@ import difflib import os +from typing import List, Optional -from nornir.core.task import Result +from nornir.core.task import Result, Task -def _read_file(file): +def _read_file(file: str) -> List[str]: if not os.path.exists(file): return [] @@ -12,33 +13,40 @@ def _read_file(file): return f.read().splitlines() -def _generate_diff(filename, content, append): +def _generate_diff(filename: str, content: str, append: bool) -> str: original = _read_file(filename) if append: c = list(original) c.extend(content.splitlines()) - content = c + new_content = c else: - content = content.splitlines() + new_content = content.splitlines() - diff = difflib.unified_diff(original, content, fromfile=filename, tofile="new") + diff = difflib.unified_diff(original, new_content, fromfile=filename, tofile="new") return "\n".join(diff) -def write_file(task, filename, content, append=False, dry_run=None): +def write_file( + task: Task, + filename: str, + content: str, + append: bool = False, + dry_run: Optional[bool] = None, +) -> Result: """ Write contents to a file (locally) Arguments: - dry_run (bool): Whether to apply changes or not - filename (``str``): file you want to write into - conteint (``str``): content you want to write - append (``bool``): whether you want to replace the contents or append to it + dry_run: Whether to apply changes or not + filename: file you want to write into + content: content you want to write + append: whether you want to replace the contents or append to it Returns: - * changed (``bool``): - * diff (``str``): unified diff + Result object with the following attributes set: + * changed (``bool``): + * diff (``str``): unified diff """ diff = _generate_diff(filename, content, append) diff --git a/nornir/plugins/tasks/networking/napalm_cli.py b/nornir/plugins/tasks/networking/napalm_cli.py index a1d75921..b6836ca2 100644 --- a/nornir/plugins/tasks/networking/napalm_cli.py +++ b/nornir/plugins/tasks/networking/napalm_cli.py @@ -1,16 +1,18 @@ -from nornir.core.task import Result +from typing import List +from nornir.core.task import Result, Task -def napalm_cli(task, commands): + +def napalm_cli(task: Task, commands: List[str]) -> Result: """ Run commands on remote devices using napalm Arguments: - commands (``list``): List of commands to execute + commands: commands to execute Returns: - :obj:`nornir.core.task.Result`: - * result (``dict``): dictionary with the result of the commands + Result object with the following attributes set: + * result (``dict``): result of the commands execution """ device = task.host.get_connection("napalm") result = device.cli(commands) diff --git a/nornir/plugins/tasks/networking/napalm_configure.py b/nornir/plugins/tasks/networking/napalm_configure.py index 88ede18c..8d674187 100644 --- a/nornir/plugins/tasks/networking/napalm_configure.py +++ b/nornir/plugins/tasks/networking/napalm_configure.py @@ -1,21 +1,27 @@ -from nornir.core.task import Result +from typing import Optional + +from nornir.core.task import Result, Task def napalm_configure( - task, dry_run=None, filename=None, configuration=None, replace=False -): + task: Task, + dry_run: Optional[bool] = None, + filename: Optional[str] = None, + configuration: Optional[str] = None, + replace: bool = False, +) -> Result: """ Loads configuration into a network devices using napalm Arguments: - dry_run (bool): Whether to apply changes or not - configuration (str): configuration to load into the device - filename (str): filename containing the configuration to load into the device - replace (bool): whether to replace or merge the configuration + dry_run: Whether to apply changes or not + filename: filename containing the configuration to load into the device + configuration: configuration to load into the device + replace: whether to replace or merge the configuration Returns: - :obj:`nornir.core.task.Result`: - * changed (``bool``): whether if the task is changing the system or not + Result object with the following attributes set: + * changed (``bool``): whether the task is changing the system or not * diff (``string``): change in the system """ device = task.host.get_connection("napalm") diff --git a/nornir/plugins/tasks/networking/napalm_get.py b/nornir/plugins/tasks/networking/napalm_get.py index 5b2185b0..49a32c12 100644 --- a/nornir/plugins/tasks/networking/napalm_get.py +++ b/nornir/plugins/tasks/networking/napalm_get.py @@ -1,14 +1,22 @@ import copy +from typing import List, Optional, Dict, Any -from nornir.core.task import Result +from nornir.core.task import Result, Task +GetterOptionsDict = Optional[Dict[str, Dict[str, Any]]] -def napalm_get(task, getters, getters_options=None, **kwargs): + +def napalm_get( + task: Task, + getters: List[str], + getters_options: GetterOptionsDict = None, + **kwargs: Any +) -> Result: """ Gather information from network devices using napalm Arguments: - getters (list of str): getters to use + getters: getters to use getters_options (dict of dicts): When passing multiple getters you pass a dictionary where the outer key is the getter name and the included dictionary represents the options to pass @@ -35,7 +43,7 @@ def napalm_get(task, getters, getters_options=None, **kwargs): > getters_options={"config": {"retrieve": "all"}}) Returns: - :obj:`nornir.core.task.Result`: + Result object with the following attributes set: * result (``dict``): dictionary with the result of the getter """ device = task.host.get_connection("napalm") diff --git a/nornir/plugins/tasks/networking/napalm_validate.py b/nornir/plugins/tasks/networking/napalm_validate.py index 1149e1a5..09234c69 100644 --- a/nornir/plugins/tasks/networking/napalm_validate.py +++ b/nornir/plugins/tasks/networking/napalm_validate.py @@ -1,7 +1,14 @@ -from nornir.core.task import Result +from typing import Optional, Dict, Any +from nornir.core.task import Result, Task +ValidationSourceData = Optional[Dict[str, Dict[str, Any]]] -def napalm_validate(task, src=None, validation_source=None): + +def napalm_validate( + task: Task, + src: Optional[str] = None, + validation_source: ValidationSourceData = None, +) -> Result: """ Gather information with napalm and validate it: @@ -9,10 +16,10 @@ def napalm_validate(task, src=None, validation_source=None): Arguments: src: file to use as validation source - validation_source (list): instead of a file data needed to validate device's state + validation_source (list): data to validate device's state Returns: - :obj:`nornir.core.task.Result`: + Result object with the following attributes set: * result (``dict``): dictionary with the result of the validation * complies (``bool``): Whether the device complies or not """ diff --git a/nornir/plugins/tasks/networking/netmiko_file_transfer.py b/nornir/plugins/tasks/networking/netmiko_file_transfer.py index 05bffed2..ee40e2ee 100644 --- a/nornir/plugins/tasks/networking/netmiko_file_transfer.py +++ b/nornir/plugins/tasks/networking/netmiko_file_transfer.py @@ -1,19 +1,22 @@ -from nornir.core.task import Result +from typing import Any +from nornir.core.task import Result, Task from netmiko import file_transfer -def netmiko_file_transfer(task, source_file, dest_file, **kwargs): +def netmiko_file_transfer( + task: Task, source_file: str, dest_file: str, **kwargs: Any +) -> Result: """ Execute Netmiko file_transfer method Arguments: - source_file(str): Source file. - dest_file(str): Destination file. - kwargs (dict, optional): Additional arguments to pass to file_transfer + source_file: Source file. + dest_file: Destination file. + kwargs: Additional arguments to pass to file_transfer Returns: - :obj:`nornir.core.task.Result`: + Result object with the following attributes set: * result (``bool``): file exists and MD5 is valid * changed (``bool``): the destination file was changed diff --git a/nornir/plugins/tasks/networking/netmiko_send_command.py b/nornir/plugins/tasks/networking/netmiko_send_command.py index 3097812b..3a2872f4 100644 --- a/nornir/plugins/tasks/networking/netmiko_send_command.py +++ b/nornir/plugins/tasks/networking/netmiko_send_command.py @@ -1,17 +1,21 @@ -from nornir.core.task import Result +from typing import Any +from nornir.core.task import Result, Task -def netmiko_send_command(task, command_string, use_timing=False, **kwargs): + +def netmiko_send_command( + task: Task, command_string: str, use_timing: bool = False, **kwargs: Any +) -> Result: """ Execute Netmiko send_command method (or send_command_timing) Arguments: - command_string(str): Command to execute on the remote network device. - use_timing(bool, optional): Set to True to switch to send_command_timing method. - kwargs (dict, optional): Additional arguments to pass to send_command method. + command_string: Command to execute on the remote network device. + use_timing: Set to True to switch to send_command_timing method. + kwargs: Additional arguments to pass to send_command method. Returns: - :obj:`nornir.core.task.Result`: + Result object with the following attributes set: * result (``dict``): dictionary with the result of the show command. """ net_connect = task.host.get_connection("netmiko") diff --git a/nornir/plugins/tasks/networking/netmiko_send_config.py b/nornir/plugins/tasks/networking/netmiko_send_config.py index e200067e..49f3eb48 100644 --- a/nornir/plugins/tasks/networking/netmiko_send_config.py +++ b/nornir/plugins/tasks/networking/netmiko_send_config.py @@ -1,19 +1,24 @@ -from __future__ import unicode_literals +from typing import Optional, Any, List +from nornir.core.task import Result, Task -from nornir.core.task import Result - -def netmiko_send_config(task, config_commands=None, config_file=None, **kwargs): +def netmiko_send_config( + task: Task, + config_commands: Optional[List[str]] = None, + config_file: Optional[str] = None, + **kwargs: Any +) -> Result: """ Execute Netmiko send_config_set method (or send_config_from_file) + Arguments: - config_commands(list, optional): Commands to configure on the remote network device. - config_file(str, optional): File to read configuration commands from. - kwargs (dict, optional): Additional arguments to pass to method. + config_commands: Commands to configure on the remote network device. + config_file: File to read configuration commands from. + kwargs: Additional arguments to pass to method. Returns: - :obj:`nornir.core.task.Result`: - * result (``dict``): dictionary showing the CLI from the configuration changes. + Result object with the following attributes set: + * result (``dict``): dictionary showing the CLI from the configuration changes """ net_connect = task.host.get_connection("netmiko") if config_commands: diff --git a/nornir/plugins/tasks/networking/tcp_ping.py b/nornir/plugins/tasks/networking/tcp_ping.py index 0d7fda15..85ab248f 100644 --- a/nornir/plugins/tasks/networking/tcp_ping.py +++ b/nornir/plugins/tasks/networking/tcp_ping.py @@ -1,22 +1,24 @@ import socket +from typing import Optional, List +from nornir.core.task import Result, Task -from nornir.core.task import Result - -def tcp_ping(task, ports, timeout=2, host=None): +def tcp_ping( + task: Task, ports: List[int], timeout: int = 2, host: Optional[str] = None +) -> Result: """ Tests connection to a tcp port and tries to establish a three way handshake. To be used for network discovery or testing. Arguments: - ports (list of int): tcp port to ping - timeout (int, optional): defaults to 0.5 + ports (list of int): tcp ports to ping + timeout (int, optional): defaults to 2 host (string, optional): defaults to ``nornir_ip`` Returns: - :obj:`nornir.core.task.Result`: + Result object with the following attributes set: * result (``dict``): Contains port numbers as keys with True/False as values """ diff --git a/nornir/plugins/tasks/text/template_file.py b/nornir/plugins/tasks/text/template_file.py index 38b70054..387fa0b9 100644 --- a/nornir/plugins/tasks/text/template_file.py +++ b/nornir/plugins/tasks/text/template_file.py @@ -1,20 +1,30 @@ +from typing import Any, Optional, Dict, Callable + from nornir.core.helpers import jinja_helper, merge_two_dicts -from nornir.core.task import Result +from nornir.core.task import Result, Task + +FiltersDict = Optional[Dict[str, Callable[..., str]]] -def template_file(task, template, path, jinja_filters=None, **kwargs): +def template_file( + task: Task, + template: str, + path: str, + jinja_filters: FiltersDict = None, + **kwargs: Any +): """ Renders contants of a file with jinja2. All the host data is available in the tempalte Arguments: - template (string): filename - path (string): path to dir with templates - jinja_filters (dict): jinja filters to enable. Defaults to nornir.config.jinja_filters + template: filename + path: path to dir with templates + jinja_filters: jinja filters to enable. Defaults to nornir.config.jinja_filters **kwargs: additional data to pass to the template Returns: - :obj:`nornir.core.task.Result`: - * result (``string``): rendered string + Result object with the following attributes set: + * result (``string``): rendered string """ jinja_filters = jinja_filters or {} or task.nornir.config.jinja_filters merged = merge_two_dicts(task.host, kwargs) diff --git a/nornir/plugins/tasks/text/template_string.py b/nornir/plugins/tasks/text/template_string.py index 6da51ecd..cec83cc2 100644 --- a/nornir/plugins/tasks/text/template_string.py +++ b/nornir/plugins/tasks/text/template_string.py @@ -1,8 +1,14 @@ +from typing import Any, Optional, Dict, Callable + from nornir.core.helpers import jinja_helper, merge_two_dicts -from nornir.core.task import Result +from nornir.core.task import Result, Task + +FiltersDict = Optional[Dict[str, Callable[..., str]]] -def template_string(task, template, jinja_filters=None, **kwargs): +def template_string( + task: Task, template: str, jinja_filters: FiltersDict = None, **kwargs: Any +): """ Renders a string with jinja2. All the host data is available in the tempalte @@ -12,8 +18,8 @@ def template_string(task, template, jinja_filters=None, **kwargs): **kwargs: additional data to pass to the template Returns: - :obj:`nornir.core.task.Result`: - * result (``string``): rendered string + Result object with the following attributes set: + * result (``string``): rendered string """ jinja_filters = jinja_filters or {} or task.nornir.config.jinja_filters merged = merge_two_dicts(task.host, kwargs) diff --git a/requirements-dev.txt b/requirements-dev.txt index 69797225..6d6e147f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,4 +6,5 @@ flake8-import-order requests-mock tox black==18.4a1; python_version >= '3.6' +mypy -r requirements.txt diff --git a/requirements.txt b/requirements.txt index 335ed1cb..bf1ff10b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ paramiko future requests ruamel.yaml +mypy_extensions diff --git a/setup.cfg b/setup.cfg index 91cd4633..945dd6ff 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,3 +20,23 @@ max_line_length = 100 [tool:pytest] addopts = --cov=nornir --cov-report=term-missing -vs python_paths = ./ + +[mypy] +# The mypy configurations: http://bit.ly/2zEl9WI +python_version = 3.6 +warn_redundant_casts = True +warn_unused_configs = True +ignore_missing_imports = True + +[mypy-nornir.plugins.*] +check_untyped_defs = True +disallow_any_generics = True +# Turn on the next flag once the whole codebase is annotated (Phase 2) +# disallow_untyped_calls = True +strict_optional = True +warn_unused_ignores = True +ignore_errors = False + +[mypy-nornir.*] +ignore_errors = True + diff --git a/tests/plugins/tasks/data/test_load_yaml.py b/tests/plugins/tasks/data/test_load_yaml.py index 9f24d6e2..fcb7364a 100644 --- a/tests/plugins/tasks/data/test_load_yaml.py +++ b/tests/plugins/tasks/data/test_load_yaml.py @@ -3,7 +3,6 @@ from nornir.plugins.tasks import data - from ruamel.yaml.scanner import ScannerError diff --git a/tox.ini b/tox.ini index 0249d564..f3714798 100644 --- a/tox.ini +++ b/tox.ini @@ -32,3 +32,11 @@ deps = basepython = python3.6 commands = pylama . + +[testenv:mypy] +deps = + -rrequirements-dev.txt + +basepython = python3.6 +commands = + mypy . From 0a1027e08dd0e6f4ffc218823bb6ce5a084db402 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Thu, 19 Jul 2018 10:54:37 +0200 Subject: [PATCH 006/109] remove ordereddict as it's not needed in py3.6+ (#190) --- nornir/plugins/tasks/data/load_json.py | 11 +++-------- nornir/plugins/tasks/data/load_yaml.py | 10 ++++------ tests/plugins/tasks/data/test_load_json.py | 11 ----------- tests/plugins/tasks/data/test_load_yaml.py | 11 ----------- 4 files changed, 7 insertions(+), 36 deletions(-) diff --git a/nornir/plugins/tasks/data/load_json.py b/nornir/plugins/tasks/data/load_json.py index 269a1da8..a0cd30e8 100644 --- a/nornir/plugins/tasks/data/load_json.py +++ b/nornir/plugins/tasks/data/load_json.py @@ -1,25 +1,22 @@ -import collections import json -from typing import Dict, MutableMapping, Any, Type +from typing import Any, Dict, MutableMapping, Type from nornir.core.task import Result, Task -def load_json(task: Task, file: str, ordered_dict: bool = False) -> Result: +def load_json(task: Task, file: str) -> Result: """ Loads a json file. Arguments: file: path to the file containing the json file to load - ordered_dict: If set to true used OrderedDict to load maps Examples: Simple example with ``ordered_dict``:: > nr.run(task=load_json, - file="mydata.json", - ordered_dict=True) + file="mydata.json") file: path to the file containing the json file to load @@ -28,8 +25,6 @@ def load_json(task: Task, file: str, ordered_dict: bool = False) -> Result: * result (``dict``): dictionary with the contents of the file """ kwargs: Dict[str, Type[MutableMapping[str, Any]]] = {} - if ordered_dict: - kwargs["object_pairs_hook"] = collections.OrderedDict with open(file, "r") as f: data = json.loads(f.read(), **kwargs) diff --git a/nornir/plugins/tasks/data/load_yaml.py b/nornir/plugins/tasks/data/load_yaml.py index 327ade3d..12ef6406 100644 --- a/nornir/plugins/tasks/data/load_yaml.py +++ b/nornir/plugins/tasks/data/load_yaml.py @@ -1,31 +1,29 @@ -import ruamel.yaml from typing import Dict from nornir.core.task import Result, Task +import ruamel.yaml + -def load_yaml(task: Task, file: str, ordered_dict: bool = False): +def load_yaml(task: Task, file: str): """ Loads a yaml file. Arguments: file: path to the file containing the yaml file to load - ordered_dict: If set to true used OrderedDict to load maps Examples: Simple example with ``ordered_dict``:: > nr.run(task=load_yaml, - file="mydata.yaml", - ordered_dict=True) + file="mydata.yaml") Returns: Result object with the following attributes set: * result (``dict``): dictionary with the contents of the file """ kwargs: Dict[str, str] = {} - kwargs["typ"] = "rt" if ordered_dict else "safe" with open(file, "r") as f: yml = ruamel.yaml.YAML(pure=True, **kwargs) data = yml.load(f.read()) diff --git a/tests/plugins/tasks/data/test_load_json.py b/tests/plugins/tasks/data/test_load_json.py index 4155478b..6e9a6e29 100644 --- a/tests/plugins/tasks/data/test_load_json.py +++ b/tests/plugins/tasks/data/test_load_json.py @@ -1,5 +1,4 @@ import os -from collections import OrderedDict from nornir.plugins.tasks import data @@ -19,16 +18,6 @@ def test_load_json(self, nornir): assert d["services"] == ["dhcp", "dns"] assert isinstance(d["a_dict"], dict) - def test_load_json_ordered_dict(self, nornir): - test_file = "{}/simple.json".format(data_dir) - result = nornir.run(data.load_json, file=test_file, ordered_dict=True) - - for h, r in result.items(): - d = r.result - assert d["env"] == "test" - assert d["services"] == ["dhcp", "dns"] - assert isinstance(d["a_dict"], OrderedDict) - def test_load_json_error_broken_file(self, nornir): test_file = "{}/broken.json".format(data_dir) results = nornir.run(data.load_json, file=test_file) diff --git a/tests/plugins/tasks/data/test_load_yaml.py b/tests/plugins/tasks/data/test_load_yaml.py index fcb7364a..08a5a5c2 100644 --- a/tests/plugins/tasks/data/test_load_yaml.py +++ b/tests/plugins/tasks/data/test_load_yaml.py @@ -1,5 +1,4 @@ import os -from collections import OrderedDict from nornir.plugins.tasks import data @@ -21,16 +20,6 @@ def test_load_yaml(self, nornir): assert d["services"] == ["dhcp", "dns"] assert isinstance(d["a_dict"], dict) - def test_load_yaml_ordered_dict(self, nornir): - test_file = "{}/simple.yaml".format(data_dir) - result = nornir.run(data.load_yaml, file=test_file, ordered_dict=True) - - for h, r in result.items(): - d = r.result - assert d["env"] == "test" - assert d["services"] == ["dhcp", "dns"] - assert isinstance(d["a_dict"], OrderedDict) - def test_load_yaml_error_broken_file(self, nornir): test_file = "{}/broken.yaml".format(data_dir) results = nornir.run(data.load_yaml, file=test_file) From 562ce4e467a0363473a4099eb7ab1ebe8116cd3b Mon Sep 17 00:00:00 2001 From: David Barroso Date: Thu, 19 Jul 2018 11:02:16 +0200 Subject: [PATCH 007/109] code reformatted with newer black (#191) --- nornir/core/__init__.py | 11 ++++++++--- nornir/core/filter.py | 5 ----- nornir/core/inventory.py | 3 ++- nornir/plugins/functions/text/__init__.py | 10 ++++++---- nornir/plugins/inventory/ansible.py | 3 --- .../plugins/tasks/connections/netmiko_connection.py | 4 +++- .../plugins/tasks/connections/paramiko_connection.py | 4 +++- requirements-dev.txt | 2 +- setup.py | 3 ++- tests/core/test_InitNornir.py | 2 -- tests/core/test_configuration.py | 1 - tests/core/test_filter.py | 1 - tests/core/test_inventory.py | 11 ++++++----- tests/core/test_multithreading.py | 1 - tests/core/test_tasks.py | 1 - tests/plugins/functions/text/test_print_result.py | 1 - tests/plugins/inventory/test_ansible.py | 1 - tests/plugins/inventory/test_nsot.py | 1 - tests/plugins/tasks/apis/test_http_method.py | 1 - tests/plugins/tasks/commands/test_command.py | 1 - tests/plugins/tasks/commands/test_remote_command.py | 1 - tests/plugins/tasks/data/test_load_json.py | 1 - tests/plugins/tasks/data/test_load_yaml.py | 1 - tests/plugins/tasks/files/test_sftp.py | 1 - tests/plugins/tasks/files/test_write_file.py | 1 - tests/plugins/tasks/networking/test_napalm_cli.py | 1 - .../plugins/tasks/networking/test_napalm_configure.py | 1 - tests/plugins/tasks/networking/test_napalm_get.py | 1 - .../plugins/tasks/networking/test_napalm_validate.py | 1 - .../tasks/networking/test_netmiko_file_transfer.py | 1 - .../tasks/networking/test_netmiko_send_command.py | 1 - .../tasks/networking/test_netmiko_send_config.py | 1 - tests/plugins/tasks/networking/test_tcp_ping.py | 1 - tests/plugins/tasks/text/test_template_file.py | 1 - tests/plugins/tasks/text/test_template_string.py | 1 - tox.ini | 2 +- 36 files changed, 32 insertions(+), 52 deletions(-) diff --git a/nornir/core/__init__.py b/nornir/core/__init__.py index ce08fe41..cd4bdf4d 100644 --- a/nornir/core/__init__.py +++ b/nornir/core/__init__.py @@ -95,7 +95,9 @@ def configure_logging(self): "handlers": {}, "loggers": {}, "root": { - "level": "CRITICAL" if self.config.logging_loggers else self.config.logging_level.upper(), # noqa + "level": "CRITICAL" + if self.config.logging_loggers + else self.config.logging_level.upper(), # noqa "handlers": [], "formatter": "simple", }, @@ -125,7 +127,8 @@ def configure_logging(self): for logger in self.config.logging_loggers: dictConfig["loggers"][logger] = { - "level": self.config.logging_level.upper(), "handlers": handlers_list + "level": self.config.logging_level.upper(), + "handlers": handlers_list, } if dictConfig["root"]["handlers"]: @@ -215,7 +218,9 @@ def run( else: result = self._run_parallel(task, run_on, num_workers, **kwargs) - raise_on_error = raise_on_error if raise_on_error is not None else self.config.raise_on_error # noqa + raise_on_error = ( + raise_on_error if raise_on_error is not None else self.config.raise_on_error + ) # noqa if raise_on_error: result.raise_on_error() else: diff --git a/nornir/core/filter.py b/nornir/core/filter.py index 8a67687b..f736cbdc 100644 --- a/nornir/core/filter.py +++ b/nornir/core/filter.py @@ -1,5 +1,4 @@ class F_OP_BASE(object): - def __init__(self, op1, op2): self.op1 = op1 self.op2 = op2 @@ -15,19 +14,16 @@ def __repr__(self): class AND(F_OP_BASE): - def __call__(self, host): return self.op1(host) and self.op2(host) class OR(F_OP_BASE): - def __call__(self, host): return self.op1(host) or self.op2(host) class F(object): - def __init__(self, **kwargs): self.filters = kwargs @@ -75,7 +71,6 @@ def _verify_rules(data, rule, value): class NOT_F(F): - def __call__(self, host): return not any( F._verify_rules(host, k.split("__"), v) for k, v in self.filters.items() diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index eda727ae..ba0e98f9 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -392,5 +392,6 @@ def to_dict(self): groups = {k: v.to_dict() for k, v in self.groups.items()} groups["defaults"] = self.defaults return { - "hosts": {k: v.to_dict() for k, v in self.hosts.items()}, "groups": groups + "hosts": {k: v.to_dict() for k, v in self.hosts.items()}, + "groups": groups, } diff --git a/nornir/plugins/functions/text/__init__.py b/nornir/plugins/functions/text/__init__.py index 10418033..85043648 100644 --- a/nornir/plugins/functions/text/__init__.py +++ b/nornir/plugins/functions/text/__init__.py @@ -44,8 +44,8 @@ def _print_individual_result( return color = _get_color(result, failed) - subtitle = "" if result.changed is None else " ** changed : {} ".format( - result.changed + subtitle = ( + "" if result.changed is None else " ** changed : {} ".format(result.changed) ) level_name = logging.getLevelName(result.severity_level) symbol = "v" if task_group else "-" @@ -78,8 +78,10 @@ def _print_result( msg = result.name print("{}{}{}{}".format(Style.BRIGHT, Fore.CYAN, msg, "*" * (80 - len(msg)))) for host, host_data in sorted(result.items()): - title = "" if host_data.changed is None else " ** changed : {} ".format( - host_data.changed + title = ( + "" + if host_data.changed is None + else " ** changed : {} ".format(host_data.changed) ) msg = "* {}{}".format(host, title) print( diff --git a/nornir/plugins/inventory/ansible.py b/nornir/plugins/inventory/ansible.py index 680c4ed0..98a1ca8c 100644 --- a/nornir/plugins/inventory/ansible.py +++ b/nornir/plugins/inventory/ansible.py @@ -31,7 +31,6 @@ class AnsibleParser(object): - def __init__(self, hostsfile: str) -> None: self.hostsfile = hostsfile self.path = os.path.dirname(hostsfile) @@ -134,7 +133,6 @@ def load_hosts_file(self) -> None: class INIParser(AnsibleParser): - @staticmethod def normalize_value(value: str) -> Union[str, int]: try: @@ -206,7 +204,6 @@ def load_hosts_file(self) -> None: class YAMLParser(AnsibleParser): - def load_hosts_file(self) -> None: with open(self.hostsfile, "r") as f: yml = ruamel.yaml.YAML(typ="rt", pure=True) diff --git a/nornir/plugins/tasks/connections/netmiko_connection.py b/nornir/plugins/tasks/connections/netmiko_connection.py index 5664eb52..02f9e815 100644 --- a/nornir/plugins/tasks/connections/netmiko_connection.py +++ b/nornir/plugins/tasks/connections/netmiko_connection.py @@ -23,7 +23,9 @@ def netmiko_connection(task: Task, **netmiko_args: Any) -> None: """ host = task.host parameters = { - "host": host.host, "username": host.username, "password": host.password + "host": host.host, + "username": host.username, + "password": host.password, } if host.ssh_port: parameters["port"] = host.ssh_port diff --git a/nornir/plugins/tasks/connections/paramiko_connection.py b/nornir/plugins/tasks/connections/paramiko_connection.py index ecdb0a89..5cb330c4 100644 --- a/nornir/plugins/tasks/connections/paramiko_connection.py +++ b/nornir/plugins/tasks/connections/paramiko_connection.py @@ -23,7 +23,9 @@ def paramiko_connection(task: Task) -> None: ssh_config.parse(f) parameters = { - "hostname": host.host, "username": host.username, "password": host.password + "hostname": host.host, + "username": host.username, + "password": host.password, } if host.ssh_port: parameters["port"] = host.ssh_port diff --git a/requirements-dev.txt b/requirements-dev.txt index 6d6e147f..0d18a6ba 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,6 +5,6 @@ pylama flake8-import-order requests-mock tox -black==18.4a1; python_version >= '3.6' +black==18.6b4 mypy -r requirements.txt diff --git a/setup.py b/setup.py index 37db35e2..80354a5e 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ test_suite="tests", platforms="any", classifiers=[ - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7" + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", ], ) diff --git a/tests/core/test_InitNornir.py b/tests/core/test_InitNornir.py index db0e4c9e..5a6273cb 100644 --- a/tests/core/test_InitNornir.py +++ b/tests/core/test_InitNornir.py @@ -14,14 +14,12 @@ def transform_func(host): class StringInventory(Inventory): - def __init__(self, *args, **kwargs): hosts = {"host1": {}, "host2": {}} super().__init__(hosts, *args, **kwargs) class Test(object): - def test_InitNornir_defaults(self): os.chdir("tests/inventory_data/") try: diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py index 1bc27399..17002fc3 100644 --- a/tests/core/test_configuration.py +++ b/tests/core/test_configuration.py @@ -11,7 +11,6 @@ class Test(object): - def test_configuration_empty(self): config = Config( config_file=os.path.join(dir_path, "empty.yaml"), diff --git a/tests/core/test_filter.py b/tests/core/test_filter.py index a8c3018d..eefc582a 100644 --- a/tests/core/test_filter.py +++ b/tests/core/test_filter.py @@ -11,7 +11,6 @@ class Test(object): - def test_simple(self): f = F(site="site1") filtered = sorted(list((inventory.filter(f).hosts.keys()))) diff --git a/tests/core/test_inventory.py b/tests/core/test_inventory.py index 600d8cfe..9387ed18 100644 --- a/tests/core/test_inventory.py +++ b/tests/core/test_inventory.py @@ -35,7 +35,6 @@ def to_sets(l): class Test(object): - def test_hosts(self): defaults = {"var4": "ALL"} g1 = Group(name="g1", var1="1", var2="2", var3="3") @@ -140,10 +139,12 @@ def test_hosts(self): def test_filtering(self): unfiltered = sorted(list(inventory.hosts.keys())) - assert ( - unfiltered - == ["dev1.group_1", "dev2.group_1", "dev3.group_2", "dev4.group_2"] - ) + assert unfiltered == [ + "dev1.group_1", + "dev2.group_1", + "dev3.group_2", + "dev4.group_2", + ] www = sorted(list(inventory.filter(role="www").hosts.keys())) assert www == ["dev1.group_1", "dev3.group_2"] diff --git a/tests/core/test_multithreading.py b/tests/core/test_multithreading.py index ad0915dc..e6ae3685 100644 --- a/tests/core/test_multithreading.py +++ b/tests/core/test_multithreading.py @@ -31,7 +31,6 @@ def verify_data_change(task): class Test(object): - def test_blocking_task_single_thread(self, nornir): t1 = datetime.datetime.now() nornir.run(blocking_task, wait=0.5, num_workers=1) diff --git a/tests/core/test_tasks.py b/tests/core/test_tasks.py index bf4001de..6ea50d84 100644 --- a/tests/core/test_tasks.py +++ b/tests/core/test_tasks.py @@ -20,7 +20,6 @@ def sub_task(task): class Test(object): - def test_task(self, nornir): result = nornir.run(commands.command, command="echo hi") assert result diff --git a/tests/plugins/functions/text/test_print_result.py b/tests/plugins/functions/text/test_print_result.py index 9d6f6ffc..7dbd3403 100644 --- a/tests/plugins/functions/text/test_print_result.py +++ b/tests/plugins/functions/text/test_print_result.py @@ -61,7 +61,6 @@ def read_data(task): class Test(object): - @wrap_cli_test(output="{}/basic_single".format(output_dir)) def test_print_basic(self, nornir): filter = nornir.filter(name="dev1.group_1") diff --git a/tests/plugins/inventory/test_ansible.py b/tests/plugins/inventory/test_ansible.py index 4d1cfe01..8a28fb42 100644 --- a/tests/plugins/inventory/test_ansible.py +++ b/tests/plugins/inventory/test_ansible.py @@ -28,7 +28,6 @@ def read(hosts_file, groups_file): class Test(object): - @pytest.mark.parametrize("case", ["ini", "yaml", "yaml2"]) def test_inventory(self, case): base_path = os.path.join(BASE_PATH, case) diff --git a/tests/plugins/inventory/test_nsot.py b/tests/plugins/inventory/test_nsot.py index 1c46ea42..c5c5d9da 100644 --- a/tests/plugins/inventory/test_nsot.py +++ b/tests/plugins/inventory/test_nsot.py @@ -29,7 +29,6 @@ def transform_function(host): class Test(object): - def test_inventory(self, requests_mock): inv = get_inv(requests_mock, "1.3.0", transform_function=transform_function) assert len(inv.hosts) == 4 diff --git a/tests/plugins/tasks/apis/test_http_method.py b/tests/plugins/tasks/apis/test_http_method.py index 6cc3b2b8..3dc79763 100644 --- a/tests/plugins/tasks/apis/test_http_method.py +++ b/tests/plugins/tasks/apis/test_http_method.py @@ -11,7 +11,6 @@ class Test(object): - def test_simple_get_text(self): url = "{}/encoding/utf8".format(BASE_URL) result = http_method(method="get", url=url) diff --git a/tests/plugins/tasks/commands/test_command.py b/tests/plugins/tasks/commands/test_command.py index a5766664..7876991b 100644 --- a/tests/plugins/tasks/commands/test_command.py +++ b/tests/plugins/tasks/commands/test_command.py @@ -8,7 +8,6 @@ def echo_hostname(task): class Test(object): - def test_command(self, nornir): result = nornir.run(echo_hostname) assert result diff --git a/tests/plugins/tasks/commands/test_remote_command.py b/tests/plugins/tasks/commands/test_remote_command.py index 945af852..9edb6ccd 100644 --- a/tests/plugins/tasks/commands/test_remote_command.py +++ b/tests/plugins/tasks/commands/test_remote_command.py @@ -3,7 +3,6 @@ class Test(object): - def test_remote_command(self, nornir): result = nornir.run(commands.remote_command, command="hostname") assert result diff --git a/tests/plugins/tasks/data/test_load_json.py b/tests/plugins/tasks/data/test_load_json.py index 6e9a6e29..f8ed31ff 100644 --- a/tests/plugins/tasks/data/test_load_json.py +++ b/tests/plugins/tasks/data/test_load_json.py @@ -7,7 +7,6 @@ class Test(object): - def test_load_json(self, nornir): test_file = "{}/simple.json".format(data_dir) result = nornir.run(data.load_json, file=test_file) diff --git a/tests/plugins/tasks/data/test_load_yaml.py b/tests/plugins/tasks/data/test_load_yaml.py index 08a5a5c2..971964a0 100644 --- a/tests/plugins/tasks/data/test_load_yaml.py +++ b/tests/plugins/tasks/data/test_load_yaml.py @@ -9,7 +9,6 @@ class Test(object): - def test_load_yaml(self, nornir): test_file = "{}/simple.yaml".format(data_dir) result = nornir.run(data.load_yaml, file=test_file) diff --git a/tests/plugins/tasks/files/test_sftp.py b/tests/plugins/tasks/files/test_sftp.py index feef3730..f9849c6f 100644 --- a/tests/plugins/tasks/files/test_sftp.py +++ b/tests/plugins/tasks/files/test_sftp.py @@ -49,7 +49,6 @@ def get_directory(task): class Test(object): - def test_sftp_put(self, nornir): result = nornir.run( files.sftp, diff --git a/tests/plugins/tasks/files/test_write_file.py b/tests/plugins/tasks/files/test_write_file.py index 6a2d0a28..618f1db1 100644 --- a/tests/plugins/tasks/files/test_write_file.py +++ b/tests/plugins/tasks/files/test_write_file.py @@ -128,7 +128,6 @@ def _test_append(task): class Test(object): - def test_write_file(self, nornir): nornir.run(_test_write_file) diff --git a/tests/plugins/tasks/networking/test_napalm_cli.py b/tests/plugins/tasks/networking/test_napalm_cli.py index 70507541..c291a7a1 100644 --- a/tests/plugins/tasks/networking/test_napalm_cli.py +++ b/tests/plugins/tasks/networking/test_napalm_cli.py @@ -12,7 +12,6 @@ class Test(object): - def test_napalm_cli(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_cli"} d = nornir.filter(name="dev3.group_2") diff --git a/tests/plugins/tasks/networking/test_napalm_configure.py b/tests/plugins/tasks/networking/test_napalm_configure.py index a699c1d7..75391d13 100644 --- a/tests/plugins/tasks/networking/test_napalm_configure.py +++ b/tests/plugins/tasks/networking/test_napalm_configure.py @@ -9,7 +9,6 @@ class Test(object): - def test_napalm_configure_change_dry_run(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_configure_change_dry_run"} configuration = "hostname changed-hostname" diff --git a/tests/plugins/tasks/networking/test_napalm_get.py b/tests/plugins/tasks/networking/test_napalm_get.py index 77e597dd..f6b3003e 100644 --- a/tests/plugins/tasks/networking/test_napalm_get.py +++ b/tests/plugins/tasks/networking/test_napalm_get.py @@ -7,7 +7,6 @@ class Test(object): - def test_napalm_getters(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters"} d = nornir.filter(name="dev3.group_2") diff --git a/tests/plugins/tasks/networking/test_napalm_validate.py b/tests/plugins/tasks/networking/test_napalm_validate.py index 213d18a3..ac1ae347 100644 --- a/tests/plugins/tasks/networking/test_napalm_validate.py +++ b/tests/plugins/tasks/networking/test_napalm_validate.py @@ -7,7 +7,6 @@ class Test(object): - def test_napalm_validate_src_ok(self, nornir): opt = {"path": THIS_DIR + "/mocked/napalm_get/test_napalm_getters"} d = nornir.filter(name="dev3.group_2") diff --git a/tests/plugins/tasks/networking/test_netmiko_file_transfer.py b/tests/plugins/tasks/networking/test_netmiko_file_transfer.py index f70c3014..8c218d36 100644 --- a/tests/plugins/tasks/networking/test_netmiko_file_transfer.py +++ b/tests/plugins/tasks/networking/test_netmiko_file_transfer.py @@ -6,7 +6,6 @@ class Test(object): - def test_netmiko_file_transfer(self, nornir): source_file = os.path.join(THIS_DIR, "data", "test_file.txt") dest_file = "test_file.txt" diff --git a/tests/plugins/tasks/networking/test_netmiko_send_command.py b/tests/plugins/tasks/networking/test_netmiko_send_command.py index 254d73d6..d7bf826c 100644 --- a/tests/plugins/tasks/networking/test_netmiko_send_command.py +++ b/tests/plugins/tasks/networking/test_netmiko_send_command.py @@ -2,7 +2,6 @@ class Test(object): - def test_explicit_netmiko_connection(self, nornir): nornir.filter(name="dev4.group_2").run(task=connections.netmiko_connection) result = nornir.filter(name="dev4.group_2").run( diff --git a/tests/plugins/tasks/networking/test_netmiko_send_config.py b/tests/plugins/tasks/networking/test_netmiko_send_config.py index edbf4bcc..3558888c 100644 --- a/tests/plugins/tasks/networking/test_netmiko_send_config.py +++ b/tests/plugins/tasks/networking/test_netmiko_send_config.py @@ -2,7 +2,6 @@ class Test(object): - def test_explicit_netmiko_connection(self, nornir): nornir.filter(name="dev4.group_2").run(task=connections.netmiko_connection) result = nornir.filter(name="dev4.group_2").run( diff --git a/tests/plugins/tasks/networking/test_tcp_ping.py b/tests/plugins/tasks/networking/test_tcp_ping.py index 920e2296..af509a5d 100644 --- a/tests/plugins/tasks/networking/test_tcp_ping.py +++ b/tests/plugins/tasks/networking/test_tcp_ping.py @@ -11,7 +11,6 @@ class Test(object): - def test_tcp_ping_port(self, nornir): filter = nornir.filter(name="dev4.group_2") result = filter.run(networking.tcp_ping, ports=65004) diff --git a/tests/plugins/tasks/text/test_template_file.py b/tests/plugins/tasks/text/test_template_file.py index 6f1c7efc..0617de36 100644 --- a/tests/plugins/tasks/text/test_template_file.py +++ b/tests/plugins/tasks/text/test_template_file.py @@ -10,7 +10,6 @@ class Test(object): - def test_template_file(self, nornir): result = nornir.run(text.template_file, template="simple.j2", path=data_dir) diff --git a/tests/plugins/tasks/text/test_template_string.py b/tests/plugins/tasks/text/test_template_string.py index c02dbd28..22c17252 100644 --- a/tests/plugins/tasks/text/test_template_string.py +++ b/tests/plugins/tasks/text/test_template_string.py @@ -25,7 +25,6 @@ class Test(object): - def test_template_string(self, nornir): result = nornir.run(text.template_string, template=simple_j2) diff --git a/tox.ini b/tox.ini index f3714798..098195b7 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ commands = py.test [testenv:black] -deps = black==18.4a1 +deps = black==18.6b4 basepython = python3.6 commands = From 19d05bf9036f5f5f4db493748b305119fa4e5caf Mon Sep 17 00:00:00 2001 From: Dmitry Figol Date: Sun, 22 Jul 2018 15:27:54 +0300 Subject: [PATCH 008/109] Fix #166 (#193) --- nornir/core/inventory.py | 22 +++++++++++++++++----- tests/core/test_inventory.py | 1 + tests/inventory_data/groups.yaml | 2 ++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index ba0e98f9..a296bb08 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -304,11 +304,23 @@ def __init__( self.defaults = defaults or {} - self.groups = groups or {} - for n, g in self.groups.items(): - if isinstance(g, dict): - g = Group(name=n, nornir=nornir, **g) - self.groups[n] = g + self.groups: Dict[str, Group] = {} + if groups is not None: + for group_name, group_details in groups.items(): + if group_details is None: + group = Group(name=group_name, nornir=nornir) + elif isinstance(group_details, dict): + group = Group(name=group_name, nornir=nornir, **group_details) + elif isinstance(group_details, Group): + group = group_details + else: + raise ValueError( + f"Parsing group {group_name}: " + f"expected dict or Group object, " + f"got {type(group_details)} instead" + ) + + self.groups[group_name] = group for group in self.groups.values(): group.groups = self._resolve_groups(group.groups) diff --git a/tests/core/test_inventory.py b/tests/core/test_inventory.py index 9387ed18..e4ab5e12 100644 --- a/tests/core/test_inventory.py +++ b/tests/core/test_inventory.py @@ -246,6 +246,7 @@ def test_to_dict(self): "groups": ["parent_group"], }, "group_2": {"name": "group_2", "site": "site2"}, + "empty_group": {"name": "empty_group"}, }, } assert inventory.filter(role="www").to_dict() == expected diff --git a/tests/inventory_data/groups.yaml b/tests/inventory_data/groups.yaml index fc8b14ae..b05fc9ed 100644 --- a/tests/inventory_data/groups.yaml +++ b/tests/inventory_data/groups.yaml @@ -17,3 +17,5 @@ group_1: group_2: site: site2 + +empty_group: From ef6645f99321d84a99a353ab953b6e4905567c8e Mon Sep 17 00:00:00 2001 From: Dmitry Figol Date: Sun, 22 Jul 2018 15:29:51 +0300 Subject: [PATCH 009/109] Remove deprecated ruamel.yaml API calls (#192) * safe_load is deprecated and was replaced with ruamel.yaml.YAML --- nornir/core/configuration.py | 3 ++- nornir/plugins/inventory/ansible.py | 6 +++--- nornir/plugins/inventory/simple.py | 5 +++-- nornir/plugins/tasks/data/load_yaml.py | 7 ++----- tests/plugins/inventory/test_ansible.py | 13 +++++++------ 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/nornir/core/configuration.py b/nornir/core/configuration.py index aa4d5f74..49973d32 100644 --- a/nornir/core/configuration.py +++ b/nornir/core/configuration.py @@ -93,7 +93,8 @@ class Config(object): def __init__(self, config_file=None, **kwargs): if config_file: with open(config_file, "r") as f: - data = ruamel.yaml.safe_load(f.read()) or {} + yml = ruamel.yaml.YAML() + data = yml.load(f) or {} else: data = {} diff --git a/nornir/plugins/inventory/ansible.py b/nornir/plugins/inventory/ansible.py index 98a1ca8c..4ff48f1f 100644 --- a/nornir/plugins/inventory/ansible.py +++ b/nornir/plugins/inventory/ansible.py @@ -104,7 +104,7 @@ def read_vars_file(element: str, path: str, is_host: bool = True) -> VarsDict: with open(filepath, "r") as f: logger.debug("AnsibleInventory: reading var file: {}".format(filepath)) - yml = ruamel.yaml.YAML(typ="rt", pure=True) + yml = ruamel.yaml.YAML() return yml.load(f) @staticmethod @@ -206,8 +206,8 @@ def load_hosts_file(self) -> None: class YAMLParser(AnsibleParser): def load_hosts_file(self) -> None: with open(self.hostsfile, "r") as f: - yml = ruamel.yaml.YAML(typ="rt", pure=True) - self.original_data = cast(AnsibleGroupsDict, yml.load(f.read())) + yml = ruamel.yaml.YAML() + self.original_data = cast(AnsibleGroupsDict, yml.load(f)) def parse(hostsfile: str) -> Tuple[HostsDict, GroupsDict]: diff --git a/nornir/plugins/inventory/simple.py b/nornir/plugins/inventory/simple.py index 9874a5d7..1946788d 100644 --- a/nornir/plugins/inventory/simple.py +++ b/nornir/plugins/inventory/simple.py @@ -122,13 +122,14 @@ def __init__( group_file: str = "groups.yaml", **kwargs: Any ) -> None: + yml = ruamel.yaml.YAML() with open(host_file, "r") as f: - hosts: HostsDict = ruamel.yaml.safe_load(f.read()) + hosts: HostsDict = yml.load(f) if group_file: if os.path.exists(group_file): with open(group_file, "r") as f: - groups: GroupsDict = ruamel.yaml.safe_load(f.read()) + groups: GroupsDict = yml.load(f) else: logging.warning("{}: doesn't exist".format(group_file)) groups = {} diff --git a/nornir/plugins/tasks/data/load_yaml.py b/nornir/plugins/tasks/data/load_yaml.py index 12ef6406..200029ea 100644 --- a/nornir/plugins/tasks/data/load_yaml.py +++ b/nornir/plugins/tasks/data/load_yaml.py @@ -1,5 +1,3 @@ -from typing import Dict - from nornir.core.task import Result, Task import ruamel.yaml @@ -23,9 +21,8 @@ def load_yaml(task: Task, file: str): Result object with the following attributes set: * result (``dict``): dictionary with the contents of the file """ - kwargs: Dict[str, str] = {} with open(file, "r") as f: - yml = ruamel.yaml.YAML(pure=True, **kwargs) - data = yml.load(f.read()) + yml = ruamel.yaml.YAML(pure=True) + data = yml.load(f) return Result(host=task.host, result=data) diff --git a/tests/plugins/inventory/test_ansible.py b/tests/plugins/inventory/test_ansible.py index 8a28fb42..735a56b9 100644 --- a/tests/plugins/inventory/test_ansible.py +++ b/tests/plugins/inventory/test_ansible.py @@ -3,8 +3,8 @@ from nornir.plugins.inventory import ansible import pytest - import ruamel.yaml +from ruamel.yaml.scanner import ScannerError BASE_PATH = os.path.join(os.path.dirname(__file__), "ansible") @@ -12,18 +12,19 @@ def save(hosts, groups, hosts_file, groups_file): yml = ruamel.yaml.YAML(typ="safe", pure=True) + yml.default_flow_style = False with open(hosts_file, "w+") as f: - f.write(yml.dump(hosts, default_flow_style=False)) + f.write(yml.dump(hosts)) with open(groups_file, "w+") as f: - f.write(yml.dump(groups, default_flow_style=False)) + f.write(yml.dump(groups)) def read(hosts_file, groups_file): yml = ruamel.yaml.YAML(typ="safe") with open(hosts_file, "r") as f: - hosts = yml.load(f.read()) + hosts = yml.load(f) with open(groups_file, "r") as f: - groups = yml.load(f.read()) + groups = yml.load(f) return hosts, groups @@ -45,5 +46,5 @@ def test_inventory(self, case): def test_parse_error(self): base_path = os.path.join(BASE_PATH, "parse_error") - with pytest.raises(ruamel.yaml.scanner.ScannerError): + with pytest.raises(ScannerError): ansible.parse(hostsfile=os.path.join(base_path, "source", "hosts")) From 50804be918fbb81a94b638f0ea180a3675543cf0 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Mon, 23 Jul 2018 17:20:33 +0200 Subject: [PATCH 010/109] Allow filtering by list, ie F(nornir_nos__in=[eos, junos]) (#197) --- nornir/core/filter.py | 2 ++ tests/core/test_filter.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/nornir/core/filter.py b/nornir/core/filter.py index f736cbdc..85bab2e8 100644 --- a/nornir/core/filter.py +++ b/nornir/core/filter.py @@ -61,6 +61,8 @@ def _verify_rules(data, rule, value): else: return getattr(data, rule[0]) == value + elif rule == ["in"]: + return data in value else: return data.get(rule[0]) == value diff --git a/tests/core/test_filter.py b/tests/core/test_filter.py index eefc582a..3ceebacf 100644 --- a/tests/core/test_filter.py +++ b/tests/core/test_filter.py @@ -111,3 +111,9 @@ def test_filtering_by_attribute_name(self): filtered = sorted(list((inventory.filter(f).hosts.keys()))) assert filtered == ["dev1.group_1"] + + def test_filtering_string_in_list(self): + f = F(nornir_nos__in=["linux", "mock"]) + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev3.group_2", "dev4.group_2"] From c46facb7d44cf9ffde927ccd0ef4bc5da9b8708e Mon Sep 17 00:00:00 2001 From: Tyler Bigler Date: Wed, 25 Jul 2018 10:41:54 -0400 Subject: [PATCH 011/109] Add initial explanation of DSL. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2e2a86c7..01eb6972 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Nornir ======= -Nornir is a pure Python automation framework intented to be used directly from Python. While most automation frameworks use their own DSL which you use to describe what you want to have done, Nornir lets you control everything from Python. +Nornir is a pure Python automation framework intented to be used directly from Python. While most automation frameworks use their own Domain Specific Language (DSL) which you use to describe what you want to have done, Nornir lets you control everything from Python. One of the benefits we want to highlight with this approach is the ease of troubleshooting, if something goes wrong you can just use your existing debug tools directly from Python (just add a line of `import pdb` & `pdb.set_trace()` and you're good to go). Doing the same using a DSL can be quite time consuming. From e4e9419c1ae47f722e3bf7624fd18743659799d8 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sat, 28 Jul 2018 11:51:08 +0200 Subject: [PATCH 012/109] implement any/all filtering for collections (#203) --- nornir/core/filter.py | 4 ++++ tests/core/test_filter.py | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/nornir/core/filter.py b/nornir/core/filter.py index 85bab2e8..f71f35df 100644 --- a/nornir/core/filter.py +++ b/nornir/core/filter.py @@ -63,6 +63,10 @@ def _verify_rules(data, rule, value): elif rule == ["in"]: return data in value + elif rule == ["any"]: + return any([x in data for x in value]) + elif rule == ["all"]: + return all([x in data for x in value]) else: return data.get(rule[0]) == value diff --git a/tests/core/test_filter.py b/tests/core/test_filter.py index 3ceebacf..3555099e 100644 --- a/tests/core/test_filter.py +++ b/tests/core/test_filter.py @@ -117,3 +117,15 @@ def test_filtering_string_in_list(self): filtered = sorted(list((inventory.filter(f).hosts.keys()))) assert filtered == ["dev3.group_2", "dev4.group_2"] + + def test_filtering_list_any(self): + f = F(nested_data__a_list__any=[1, 3]) + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev1.group_1", "dev2.group_1"] + + def test_filtering_list_all(self): + f = F(nested_data__a_list__all=[1, 2]) + filtered = sorted(list((inventory.filter(f).hosts.keys()))) + + assert filtered == ["dev1.group_1"] From 49d1a8b6c53bbaaf3afa09a23de174d2dbe68c5f Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sat, 28 Jul 2018 16:58:45 +0200 Subject: [PATCH 013/109] Docs (#212) * add readthedocs yaml file * minor rearrangement * add missing data class --- .readthedocs.yml | 11 +++++++++++ docs/ref/api/configuration.rst | 4 +++- docs/ref/api/exceptions.rst | 20 +++++++++++++++++++- docs/ref/api/index.rst | 2 +- docs/ref/api/nornir.rst | 21 ++++++++++++--------- docs/ref/api/task.rst | 11 +++++++---- 6 files changed, 53 insertions(+), 16 deletions(-) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..155a1624 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,11 @@ +--- +build: + image: latest + +python: + version: 3.6 + pip_install: true + +formats: [] + +requirements_file: docs/requirements.txt diff --git a/docs/ref/api/configuration.rst b/docs/ref/api/configuration.rst index 5bbd3784..d729c447 100644 --- a/docs/ref/api/configuration.rst +++ b/docs/ref/api/configuration.rst @@ -1,6 +1,8 @@ Configuration -############# +============= +Configuration +------------- .. autoclass:: nornir.core.configuration.Config :members: diff --git a/docs/ref/api/exceptions.rst b/docs/ref/api/exceptions.rst index d3f92a9c..74b37c5a 100644 --- a/docs/ref/api/exceptions.rst +++ b/docs/ref/api/exceptions.rst @@ -1,7 +1,25 @@ Exceptions ========== -.. automodule:: nornir.core.exceptions +CommandError +------------ + +.. autoclass:: nornir.core.exceptions.CommandError + :members: + :undoc-members: + + +NornirExecutionError +-------------------- + +.. autoclass:: nornir.core.exceptions.NornirExecutionError + :members: + :undoc-members: + +NornirSubTaskError +------------------ + +.. autoclass:: nornir.core.exceptions.NornirSubTaskError :members: :undoc-members: diff --git a/docs/ref/api/index.rst b/docs/ref/api/index.rst index d2f5214b..895bc310 100644 --- a/docs/ref/api/index.rst +++ b/docs/ref/api/index.rst @@ -2,7 +2,7 @@ Nornir API Reference ===================== .. toctree:: - :maxdepth: 1 + :maxdepth: 2 :caption: Nornir API nornir diff --git a/docs/ref/api/nornir.rst b/docs/ref/api/nornir.rst index 7af32e14..c1b97d30 100644 --- a/docs/ref/api/nornir.rst +++ b/docs/ref/api/nornir.rst @@ -1,18 +1,21 @@ -Data -#### +Core +==== -.. autoclass:: nornir.core.Data - :members: - :undoc-members: +InitNornir +---------- + +.. automethod:: nornir.core.InitNornir Nornir -####### +------ .. autoclass:: nornir.core.Nornir :members: :undoc-members: -InitNornir -########### +Data +---- -.. automethod:: nornir.core.InitNornir +.. autoclass:: nornir.core.Data + :members: + :undoc-members: diff --git a/docs/ref/api/task.rst b/docs/ref/api/task.rst index 5a876de7..51fc034b 100644 --- a/docs/ref/api/task.rst +++ b/docs/ref/api/task.rst @@ -1,26 +1,29 @@ +Task and Results +================ + Task -#### +---- .. autoclass:: nornir.core.task.Task :members: :undoc-members: Result -###### +------ .. autoclass:: nornir.core.task.Result :members: :undoc-members: AggregatedResult -################ +---------------- .. autoclass:: nornir.core.task.AggregatedResult :members: :undoc-members: MultiResult -################ +----------- .. autoclass:: nornir.core.task.MultiResult :members: From 2084658698a77daaaa54d1c3f948fb00bff150e3 Mon Sep 17 00:00:00 2001 From: Dmitry Figol Date: Mon, 30 Jul 2018 19:53:17 +0200 Subject: [PATCH 014/109] Add AnsibleInventory support for host_vars and group_vars with *.yml and *.yaml extensions (#214) Fix #213 --- nornir/plugins/inventory/ansible.py | 39 ++++++++++++------- .../group_vars/{dbservers => dbservers.yml} | 0 .../{two.example.com => two.example.com.yaml} | 0 3 files changed, 25 insertions(+), 14 deletions(-) rename tests/plugins/inventory/ansible/yaml/source/group_vars/{dbservers => dbservers.yml} (100%) rename tests/plugins/inventory/ansible/yaml/source/host_vars/{two.example.com => two.example.com.yaml} (100%) diff --git a/nornir/plugins/inventory/ansible.py b/nornir/plugins/inventory/ansible.py index 4ff48f1f..3d233dca 100644 --- a/nornir/plugins/inventory/ansible.py +++ b/nornir/plugins/inventory/ansible.py @@ -6,15 +6,19 @@ import logging import os from collections import defaultdict +from pathlib import Path from typing import Dict, Any, Tuple, Optional, cast, Union, MutableMapping, DefaultDict -from mypy_extensions import TypedDict import ruamel.yaml +from mypy_extensions import TypedDict from ruamel.yaml.scanner import ScannerError from ruamel.yaml.composer import ComposerError from nornir.core.inventory import Inventory, VarsDict, GroupsDict, HostsDict +VARS_FILENAME_EXTENSIONS = ["", ".yml", ".yaml"] + +YAML = ruamel.yaml.YAML(typ="safe") logger = logging.getLogger("nornir") @@ -93,19 +97,27 @@ def sort_groups(self) -> None: @staticmethod def read_vars_file(element: str, path: str, is_host: bool = True) -> VarsDict: - subdir = "host_vars" if is_host else "group_vars" - filepath = os.path.join(path, subdir, element) - - if not os.path.exists(filepath): + sub_dir = "host_vars" if is_host else "group_vars" + vars_dir = Path(path) / sub_dir + if vars_dir.is_dir(): + vars_file_base = vars_dir / element + for extension in VARS_FILENAME_EXTENSIONS: + vars_file = vars_file_base.with_suffix( + vars_file_base.suffix + extension + ) + if vars_file.is_file(): + with open(vars_file) as f: + logger.debug( + "AnsibleInventory: reading var file: %s", vars_file + ) + return YAML.load(f) logger.debug( - "AnsibleInventory: var file doesn't exist: {}".format(filepath) + "AnsibleInventory: no vars file was found with the path %s " + "and one of the supported extensions: %s", + vars_file_base, + VARS_FILENAME_EXTENSIONS, ) - return {} - - with open(filepath, "r") as f: - logger.debug("AnsibleInventory: reading var file: {}".format(filepath)) - yml = ruamel.yaml.YAML() - return yml.load(f) + return {} @staticmethod def map_nornir_vars(obj: VarsDict): @@ -206,8 +218,7 @@ def load_hosts_file(self) -> None: class YAMLParser(AnsibleParser): def load_hosts_file(self) -> None: with open(self.hostsfile, "r") as f: - yml = ruamel.yaml.YAML() - self.original_data = cast(AnsibleGroupsDict, yml.load(f)) + self.original_data = cast(AnsibleGroupsDict, YAML.load(f)) def parse(hostsfile: str) -> Tuple[HostsDict, GroupsDict]: diff --git a/tests/plugins/inventory/ansible/yaml/source/group_vars/dbservers b/tests/plugins/inventory/ansible/yaml/source/group_vars/dbservers.yml similarity index 100% rename from tests/plugins/inventory/ansible/yaml/source/group_vars/dbservers rename to tests/plugins/inventory/ansible/yaml/source/group_vars/dbservers.yml diff --git a/tests/plugins/inventory/ansible/yaml/source/host_vars/two.example.com b/tests/plugins/inventory/ansible/yaml/source/host_vars/two.example.com.yaml similarity index 100% rename from tests/plugins/inventory/ansible/yaml/source/host_vars/two.example.com rename to tests/plugins/inventory/ansible/yaml/source/host_vars/two.example.com.yaml From eba2a7103b45ea660a0e2d7217878a1a16871d55 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Mon, 30 Jul 2018 20:14:29 +0200 Subject: [PATCH 015/109] Adding inventory plugin for netbox (#180) * Attempt at creating a custom inventory using NetBox as a backend (#162) Added inventory plugin for NetBox * fix tests for netbox * address comments * fix comment * use raise_on_status instead of try...except.. block * make black happy --- nornir/plugins/inventory/netbox.py | 71 +++++++ .../inventory/netbox/2.3.5/expected.json | 40 ++++ .../2.3.5/expected_transform_function.json | 40 ++++ .../netbox/2.3.5/mocked/devices.json | 193 ++++++++++++++++++ tests/plugins/inventory/test_netbox.py | 42 ++++ 5 files changed, 386 insertions(+) create mode 100644 nornir/plugins/inventory/netbox.py create mode 100644 tests/plugins/inventory/netbox/2.3.5/expected.json create mode 100644 tests/plugins/inventory/netbox/2.3.5/expected_transform_function.json create mode 100644 tests/plugins/inventory/netbox/2.3.5/mocked/devices.json create mode 100644 tests/plugins/inventory/test_netbox.py diff --git a/nornir/plugins/inventory/netbox.py b/nornir/plugins/inventory/netbox.py new file mode 100644 index 00000000..6b95d8a8 --- /dev/null +++ b/nornir/plugins/inventory/netbox.py @@ -0,0 +1,71 @@ +import os +from builtins import super + +from nornir.core.inventory import Inventory + +import requests + + +class NBInventory(Inventory): + + def __init__( + self, + nb_url=None, + nb_token=None, + use_slugs=True, + flatten_custom_fields=True, + **kwargs + ): + + nb_url = nb_url or os.environ.get("NB_URL", "http://localhost:8080") + nb_token = nb_token or os.environ.get( + "NB_TOKEN", "0123456789abcdef0123456789abcdef01234567" + ) + headers = {"Authorization": "Token {}".format(nb_token)} + + # Create dict of hosts using 'devices' from NetBox + r = requests.get("{}/api/dcim/devices/?limit=0".format(nb_url), headers=headers) + r.raise_for_status() + nb_devices = r.json() + + devices = {} + for d in nb_devices["results"]: + + # Create temporary dict + temp = {} + + # Add value for IP address + if d.get("primary_ip", {}): + temp["nornir_host"] = d["primary_ip"]["address"].split("/")[0] + + # Add values that don't have an option for 'slug' + temp["serial"] = d["serial"] + temp["vendor"] = d["device_type"]["manufacturer"]["name"] + temp["asset_tag"] = d["asset_tag"] + + if flatten_custom_fields: + for cf, value in d["custom_fields"].items(): + temp[cf] = value + else: + temp["custom_fields"] = d["custom_fields"] + + # Add values that do have an option for 'slug' + if use_slugs: + temp["site"] = d["site"]["slug"] + temp["role"] = d["device_role"]["slug"] + temp["model"] = d["device_type"]["slug"] + + # Attempt to add 'platform' based of value in 'slug' + temp["nornir_nos"] = d["platform"]["slug"] if d["platform"] else None + + else: + temp["site"] = d["site"]["name"] + temp["role"] = d["device_role"] + temp["model"] = d["device_type"] + temp["nornir_nos"] = d["platform"] + + # Assign temporary dict to outer dict + devices[d["name"]] = temp + + # Pass the data back to the parent class + super().__init__(devices, None, **kwargs) diff --git a/tests/plugins/inventory/netbox/2.3.5/expected.json b/tests/plugins/inventory/netbox/2.3.5/expected.json new file mode 100644 index 00000000..f6c57ee1 --- /dev/null +++ b/tests/plugins/inventory/netbox/2.3.5/expected.json @@ -0,0 +1,40 @@ +{ + "hosts": { + "1-Core": { + "name": "1-Core", + "nornir_host": "10.0.1.1", + "serial": "", + "vendor": "Juniper", + "asset_tag": null, + "site": "sunnyvale-ca", + "role": "rt", + "model": "mx480", + "nornir_nos": null + }, + "2-Distribution": { + "name": "2-Distribution", + "nornir_host": "172.16.2.1", + "serial": "", + "vendor": "Juniper", + "asset_tag": null, + "site": "sunnyvale-ca", + "role": "rt", + "model": "ex4550-32f", + "nornir_nos": null + }, + "3-Access": { + "name": "3-Access", + "nornir_host": "192.168.3.1", + "serial": "", + "vendor": "Cisco", + "asset_tag": null, + "site": "san-jose-ca", + "role": "sw", + "model": "3650-48tq-l", + "nornir_nos": null + } + }, + "groups": { + "defaults": {} + } +} diff --git a/tests/plugins/inventory/netbox/2.3.5/expected_transform_function.json b/tests/plugins/inventory/netbox/2.3.5/expected_transform_function.json new file mode 100644 index 00000000..ba70e1a2 --- /dev/null +++ b/tests/plugins/inventory/netbox/2.3.5/expected_transform_function.json @@ -0,0 +1,40 @@ +{ + "hosts": { + "1-Core": { + "name": "1-Core", + "nornir_host": "10.0.1.1", + "serial": "", + "vendor": "Juniper", + "asset_tag": null, + "site": "sunnyvale-ca", + "role": "rt", + "model": "mx480", + "nornir_nos": "junos" + }, + "2-Distribution": { + "name": "2-Distribution", + "nornir_host": "172.16.2.1", + "serial": "", + "vendor": "Juniper", + "asset_tag": null, + "site": "sunnyvale-ca", + "role": "rt", + "model": "ex4550-32f", + "nornir_nos": "junos" + }, + "3-Access": { + "name": "3-Access", + "nornir_host": "192.168.3.1", + "serial": "", + "vendor": "Cisco", + "asset_tag": null, + "site": "san-jose-ca", + "role": "sw", + "model": "3650-48tq-l", + "nornir_nos": "ios" + } + }, + "groups": { + "defaults": {} + } +} diff --git a/tests/plugins/inventory/netbox/2.3.5/mocked/devices.json b/tests/plugins/inventory/netbox/2.3.5/mocked/devices.json new file mode 100644 index 00000000..cb707cd3 --- /dev/null +++ b/tests/plugins/inventory/netbox/2.3.5/mocked/devices.json @@ -0,0 +1,193 @@ +{ + "count": 3, + "next": "http://localhost:8080/api/dcim/devices/?limit=0&offset=0", + "previous": null, + "results": [ + { + "id": 1, + "name": "1-Core", + "display_name": "1-Core", + "device_type": { + "id": 11, + "url": "http://localhost:8080/api/dcim/device-types/11/", + "manufacturer": { + "id": 3, + "url": "http://localhost:8080/api/dcim/manufacturers/3/", + "name": "Juniper", + "slug": "juniper" + }, + "model": "MX480", + "slug": "mx480" + }, + "device_role": { + "id": 1, + "url": "http://localhost:8080/api/dcim/device-roles/1/", + "name": "Router", + "slug": "rt" + }, + "tenant": null, + "platform": null, + "serial": "", + "asset_tag": null, + "site": { + "id": 3, + "url": "http://localhost:8080/api/dcim/sites/3/", + "name": "Sunnyvale, CA", + "slug": "sunnyvale-ca" + }, + "rack": null, + "position": null, + "face": null, + "parent_device": null, + "status": { + "value": 1, + "label": "Active" + }, + "primary_ip": { + "id": 1, + "url": "http://localhost:8080/api/ipam/ip-addresses/1/", + "family": 4, + "address": "10.0.1.1/32" + }, + "primary_ip4": { + "id": 1, + "url": "http://localhost:8080/api/ipam/ip-addresses/1/", + "family": 4, + "address": "10.0.1.1/32" + }, + "primary_ip6": null, + "cluster": null, + "virtual_chassis": null, + "vc_position": null, + "vc_priority": null, + "comments": "", + "custom_fields": {}, + "created": "2018-07-12", + "last_updated": "2018-07-12T11:53:54.742412Z" + }, + { + "id": 2, + "name": "2-Distribution", + "display_name": "2-Distribution", + "device_type": { + "id": 9, + "url": "http://localhost:8080/api/dcim/device-types/9/", + "manufacturer": { + "id": 3, + "url": "http://localhost:8080/api/dcim/manufacturers/3/", + "name": "Juniper", + "slug": "juniper" + }, + "model": "EX4550-32F", + "slug": "ex4550-32f" + }, + "device_role": { + "id": 1, + "url": "http://localhost:8080/api/dcim/device-roles/1/", + "name": "Router", + "slug": "rt" + }, + "tenant": null, + "platform": null, + "serial": "", + "asset_tag": null, + "site": { + "id": 3, + "url": "http://localhost:8080/api/dcim/sites/3/", + "name": "Sunnyvale, CA", + "slug": "sunnyvale-ca" + }, + "rack": null, + "position": null, + "face": null, + "parent_device": null, + "status": { + "value": 1, + "label": "Active" + }, + "primary_ip": { + "id": 2, + "url": "http://localhost:8080/api/ipam/ip-addresses/2/", + "family": 4, + "address": "172.16.2.1/32" + }, + "primary_ip4": { + "id": 2, + "url": "http://localhost:8080/api/ipam/ip-addresses/2/", + "family": 4, + "address": "172.16.2.1/32" + }, + "primary_ip6": null, + "cluster": null, + "virtual_chassis": null, + "vc_position": null, + "vc_priority": null, + "comments": "", + "custom_fields": {}, + "created": "2018-07-12", + "last_updated": "2018-07-12T11:53:54.802934Z" + }, + { + "id": 3, + "name": "3-Access", + "display_name": "3-Access", + "device_type": { + "id": 2, + "url": "http://localhost:8080/api/dcim/device-types/2/", + "manufacturer": { + "id": 2, + "url": "http://localhost:8080/api/dcim/manufacturers/2/", + "name": "Cisco", + "slug": "cisco" + }, + "model": "3650-48TQ-L", + "slug": "3650-48tq-l" + }, + "device_role": { + "id": 2, + "url": "http://localhost:8080/api/dcim/device-roles/2/", + "name": "Switch", + "slug": "sw" + }, + "tenant": null, + "platform": null, + "serial": "", + "asset_tag": null, + "site": { + "id": 2, + "url": "http://localhost:8080/api/dcim/sites/2/", + "name": "San Jose, CA", + "slug": "san-jose-ca" + }, + "rack": null, + "position": null, + "face": null, + "parent_device": null, + "status": { + "value": 1, + "label": "Active" + }, + "primary_ip": { + "id": 3, + "url": "http://localhost:8080/api/ipam/ip-addresses/3/", + "family": 4, + "address": "192.168.3.1/32" + }, + "primary_ip4": { + "id": 3, + "url": "http://localhost:8080/api/ipam/ip-addresses/3/", + "family": 4, + "address": "192.168.3.1/32" + }, + "primary_ip6": null, + "cluster": null, + "virtual_chassis": null, + "vc_position": null, + "vc_priority": null, + "comments": "", + "custom_fields": {}, + "created": "2018-07-12", + "last_updated": "2018-07-12T11:53:54.866133Z" + } + ] +} diff --git a/tests/plugins/inventory/test_netbox.py b/tests/plugins/inventory/test_netbox.py new file mode 100644 index 00000000..c48ba17b --- /dev/null +++ b/tests/plugins/inventory/test_netbox.py @@ -0,0 +1,42 @@ +import json +import os + +from nornir.plugins.inventory import netbox + +# We need import below to load fixtures +import pytest # noqa + + +BASE_PATH = os.path.join(os.path.dirname(__file__), "netbox") + + +def get_inv(requests_mock, case, **kwargs): + with open("{}/{}/mocked/devices.json".format(BASE_PATH, case), "r") as f: + requests_mock.get( + "http://localhost:8080/api/dcim/devices/?limit=0", + json=json.load(f), + headers={"Content-type": "application/json"}, + ) + return netbox.NBInventory(**kwargs) + + +def transform_function(host): + vendor_map = {"Cisco": "ios", "Juniper": "junos"} + host["nornir_nos"] = vendor_map[host["vendor"]] + + +class Test(object): + + def test_inventory(self, requests_mock): + inv = get_inv(requests_mock, "2.3.5") + with open("{}/{}/expected.json".format(BASE_PATH, "2.3.5"), "r") as f: + expected = json.load(f) + assert expected == inv.to_dict() + + def test_transform_function(self, requests_mock): + inv = get_inv(requests_mock, "2.3.5", transform_function=transform_function) + with open( + "{}/{}/expected_transform_function.json".format(BASE_PATH, "2.3.5"), "r" + ) as f: + expected = json.load(f) + assert expected == inv.to_dict() From 371bbeed278a2eb5219773efc3a0251fcf4d396c Mon Sep 17 00:00:00 2001 From: David Barroso Date: Mon, 30 Jul 2018 20:15:21 +0200 Subject: [PATCH 016/109] Refactor connection plugins (#195) * Refactor connection plugins * migrate tasks to new connection plugin mechanism * update mypy conf * update docs and tests * replace Connections.__del__ with a context manager for Nornir * Roll setup.py to indicate version 2.x.x (#199) * Roll setup.py to indicate version 2.x.x * Fix issue with None timeout value * addressing various minor comments * handle better opening/closing connections * add tests * fix and test context manager * connection-related exceptions should inherit from ConnectionException --- docs/conf.py | 7 + docs/howto/writing_a_connection_plugin.rst | 4 + docs/howto/writing_a_connection_task.rst | 24 --- docs/plugins/connections/index.rst | 20 +++ docs/plugins/index.rst | 5 +- docs/plugins/tasks/index.rst | 1 - .../tasks => ref/api}/connections.rst | 2 +- docs/ref/api/index.rst | 7 +- nornir/core/__init__.py | 27 +++- nornir/core/connections.py | 58 ++++++++ nornir/core/exceptions.py | 13 ++ nornir/core/inventory.py | 138 ++++++++++++++---- nornir/plugins/connections/__init__.py | 16 ++ nornir/plugins/connections/napalm.py | 51 +++++++ nornir/plugins/connections/netmiko.py | 56 +++++++ nornir/plugins/connections/paramiko.py | 71 +++++++++ .../plugins/tasks/commands/remote_command.py | 3 +- nornir/plugins/tasks/connections/__init__.py | 12 -- .../tasks/connections/napalm_connection.py | 59 -------- .../tasks/connections/netmiko_connection.py | 42 ------ .../tasks/connections/paramiko_connection.py | 50 ------- nornir/plugins/tasks/networking/napalm_get.py | 2 +- .../tasks/networking/napalm_validate.py | 3 +- .../tasks/networking/netmiko_file_transfer.py | 3 +- .../tasks/networking/netmiko_send_config.py | 3 +- setup.cfg | 9 ++ setup.py | 2 +- tests/core/test_connections.py | 96 ++++++++++++ .../tasks/networking/test_napalm_cli.py | 17 ++- .../tasks/networking/test_napalm_configure.py | 25 +++- .../tasks/networking/test_napalm_get.py | 27 +++- .../tasks/networking/test_napalm_validate.py | 21 ++- .../networking/test_netmiko_send_command.py | 11 +- .../networking/test_netmiko_send_config.py | 11 +- 34 files changed, 623 insertions(+), 273 deletions(-) create mode 100644 docs/howto/writing_a_connection_plugin.rst delete mode 100644 docs/howto/writing_a_connection_task.rst create mode 100644 docs/plugins/connections/index.rst rename docs/{plugins/tasks => ref/api}/connections.rst (53%) create mode 100644 nornir/core/connections.py create mode 100644 nornir/plugins/connections/__init__.py create mode 100644 nornir/plugins/connections/napalm.py create mode 100644 nornir/plugins/connections/netmiko.py create mode 100644 nornir/plugins/connections/paramiko.py delete mode 100644 nornir/plugins/tasks/connections/__init__.py delete mode 100644 nornir/plugins/tasks/connections/napalm_connection.py delete mode 100644 nornir/plugins/tasks/connections/netmiko_connection.py delete mode 100644 nornir/plugins/tasks/connections/paramiko_connection.py create mode 100644 tests/core/test_connections.py diff --git a/docs/conf.py b/docs/conf.py index baf2ddff..25cab2c7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -186,9 +186,16 @@ def build_configuration_parameters(app): f.write(rendered_template) +def skip_slots(app, what, name, obj, skip, options): + if obj.__class__.__name__ == "member_descriptor": + return True + return None + + def setup(app): """Map methods to states of the documentation build.""" app.connect("builder-inited", build_configuration_parameters) + app.connect("autodoc-skip-member", skip_slots) app.add_stylesheet("css/custom.css") diff --git a/docs/howto/writing_a_connection_plugin.rst b/docs/howto/writing_a_connection_plugin.rst new file mode 100644 index 00000000..12268c7f --- /dev/null +++ b/docs/howto/writing_a_connection_plugin.rst @@ -0,0 +1,4 @@ +Writing a connection plugin +########################### + +See :obj:`nornir.core.connections.ConnectionPlugin` and `this `_. diff --git a/docs/howto/writing_a_connection_task.rst b/docs/howto/writing_a_connection_task.rst deleted file mode 100644 index e5601886..00000000 --- a/docs/howto/writing_a_connection_task.rst +++ /dev/null @@ -1,24 +0,0 @@ -Writing a connection task -######################### - -Connection tasks are tasks that establish a connection with a device to provide some sort of reusable mechanism to interact with it. You can find some examples of connections tasks in the :doc:`../plugins/tasks/connections` section. - -Writing a connection task is no different from writing a regular task. The only difference is that the task will have to establish the connection and assign it to the device. - -A continuation you can see a simplified version of the ``paramiko_connection`` connection task as an example:: - - def paramiko_connection(task=None): - host = task.host - - client = paramiko.SSHClient() - - parameters = { - "hostname": host.host, - "username": host.username, - "password": host.password, - "port": host.ssh_port, - } - client.connect(**parameters) - host.connections["paramiko"] = client - -Note the last line where the connection is assigned to the host. Subsequent tasks will be able to retrieve this connection by host calling ``host.get_connection("paramiko")`` diff --git a/docs/plugins/connections/index.rst b/docs/plugins/connections/index.rst new file mode 100644 index 00000000..1d8f5330 --- /dev/null +++ b/docs/plugins/connections/index.rst @@ -0,0 +1,20 @@ +Connections +=========== + +NAPALM +------ + +.. automodule:: nornir.plugins.connections.napalm + :members: + +Netmiko +------- + +.. automodule:: nornir.plugins.connections.netmiko + :members: + +Paramiko +-------- + +.. automodule:: nornir.plugins.connections.paramiko + :members: diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 02f8beb5..7b7686b6 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -3,9 +3,8 @@ Plugins .. toctree:: :maxdepth: 2 - :caption: Plugins - tasks/index + connections/index functions/index inventory/index - + tasks/index diff --git a/docs/plugins/tasks/index.rst b/docs/plugins/tasks/index.rst index 1c4704ce..45db3ea4 100644 --- a/docs/plugins/tasks/index.rst +++ b/docs/plugins/tasks/index.rst @@ -7,7 +7,6 @@ Tasks apis commands - connections data files networking diff --git a/docs/plugins/tasks/connections.rst b/docs/ref/api/connections.rst similarity index 53% rename from docs/plugins/tasks/connections.rst rename to docs/ref/api/connections.rst index 77619b07..17ede1eb 100644 --- a/docs/plugins/tasks/connections.rst +++ b/docs/ref/api/connections.rst @@ -1,6 +1,6 @@ Connections =========== -.. automodule:: nornir.plugins.tasks.connections +.. automodule:: nornir.core.connections :members: :undoc-members: diff --git a/docs/ref/api/index.rst b/docs/ref/api/index.rst index 895bc310..34e65ad6 100644 --- a/docs/ref/api/index.rst +++ b/docs/ref/api/index.rst @@ -4,9 +4,6 @@ Nornir API Reference .. toctree:: :maxdepth: 2 :caption: Nornir API + :glob: - nornir - configuration - inventory - task - exceptions + * diff --git a/nornir/core/__init__.py b/nornir/core/__init__.py index cd4bdf4d..8df1d343 100644 --- a/nornir/core/__init__.py +++ b/nornir/core/__init__.py @@ -1,10 +1,12 @@ import logging import logging.config from multiprocessing.dummy import Pool +from typing import Type from nornir.core.configuration import Config +from nornir.core.connections import ConnectionPlugin from nornir.core.task import AggregatedResult, Task -from nornir.plugins.tasks import connections +from nornir.plugins import connections class Data(object): @@ -14,10 +16,12 @@ class Data(object): Attributes: failed_hosts (list): Hosts that have failed to run a task properly + available_connections (dict): Dictionary holding available connection plugins """ def __init__(self): self.failed_hosts = set() + self.available_connections = connections.available_connections def recover_host(self, host): """Remove ``host`` from list of failed hosts.""" @@ -51,7 +55,6 @@ class Nornir(object): data(:obj:`nornir.core.Data`): shared data amongst different iterations of nornir dry_run(``bool``): Whether if we are testing the changes or not config (:obj:`nornir.core.configuration.Config`): Configuration parameters - available_connections (``dict``): dict of connection types are available """ def __init__( @@ -79,9 +82,13 @@ def __init__( self.configure_logging() if available_connections is not None: - self.available_connections = available_connections - else: - self.available_connections = connections.available_connections + self.data.available_connections = available_connections + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close_connections(on_good=True, on_failed=True) @property def dry_run(self): @@ -231,6 +238,16 @@ def to_dict(self): """ Return a dictionary representing the object. """ return {"data": self.data.to_dict(), "inventory": self.inventory.to_dict()} + def get_connection_type(self, connection: str) -> Type[ConnectionPlugin]: + """Returns the class for the given connection type.""" + return self.data.available_connections[connection] + + def close_connections(self, on_good=True, on_failed=False): + def close_connections_task(task): + task.host.close_connections() + + self.run(task=close_connections_task, on_good=on_good, on_failed=on_failed) + def InitNornir(config_file="", dry_run=False, **kwargs): """ diff --git a/nornir/core/connections.py b/nornir/core/connections.py new file mode 100644 index 00000000..0763512b --- /dev/null +++ b/nornir/core/connections.py @@ -0,0 +1,58 @@ +from abc import ABC, abstractmethod +from typing import Any, Dict, NoReturn, Optional + + +from nornir.core.configuration import Config + + +class ConnectionPlugin(ABC): + """ + Connection plugins have to inherit from this class and provide implementations + for both the :meth:`open` and :meth:`close` methods. + + Attributes: + connection: Underlying connection. Populated by :meth:`open`. + state: Dictionary to hold any data that needs to be shared between + the connection plugin and the plugin tasks using this connection. + """ + + __slots__ = ("connection", "state") + + def __init__(self) -> None: + self.connection: Any = UnestablishedConnection() + self.state: Dict[str, Any] = {} + + @abstractmethod + def open( + self, + hostname: str, + username: str, + password: str, + ssh_port: int, + network_api_port: int, + operating_system: str, + nos: str, + connection_options: Optional[Dict[str, Any]] = None, + configuration: Optional[Config] = None, + ) -> None: + """ + Connect to the device and populate the attribute :attr:`connection` with + the underlying connection + """ + pass + + @abstractmethod + def close(self) -> None: + """Close the connection with the device""" + pass + + +class UnestablishedConnection(object): + def close(self) -> NoReturn: + raise ValueError("Connection not established") + + disconnect = close + + +class Connections(Dict[str, ConnectionPlugin]): + pass diff --git a/nornir/core/exceptions.py b/nornir/core/exceptions.py index e3a05f5d..cbceb400 100644 --- a/nornir/core/exceptions.py +++ b/nornir/core/exceptions.py @@ -1,6 +1,19 @@ from builtins import super +class ConnectionException(Exception): + def __init__(self, connection): + self.connection = connection + + +class ConnectionAlreadyOpen(ConnectionException): + pass + + +class ConnectionNotOpen(ConnectionException): + pass + + class CommandError(Exception): """ Raised when there is a command error. diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index a296bb08..d1f929d7 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -1,6 +1,10 @@ -from typing import Dict, Any - import getpass +from typing import Any, Dict, Optional + +from nornir.core.configuration import Config +from nornir.core.connections import Connections +from nornir.core.exceptions import ConnectionAlreadyOpen, ConnectionNotOpen + VarsDict = Dict[str, Any] HostsDict = Dict[str, VarsDict] @@ -68,9 +72,8 @@ def __init__(self, name, groups=None, nornir=None, defaults=None, **kwargs): self.groups = groups or [] self.data = {} self.data["name"] = name - self.connections = {} + self.connections = Connections() self.defaults = defaults or {} - self._ssh_forward_agent = False if len(self.groups): if isinstance(groups[0], str): @@ -230,43 +233,126 @@ def nos(self): """Network OS the device is running. Defaults to ``nornir_nos``.""" return self.get("nornir_nos") - def get_connection(self, connection): + def get_connection(self, connection: str) -> Any: """ The function of this method is twofold: 1. If an existing connection is already established for the given type return it - 2. If non exists, establish a new connection of that type with default parameters - and return it + 2. If none exists, establish a new connection of that type with default parameters + and return it Raises: - AttributeError: if it's unknown how to establish a connection for the given - type + AttributeError: if it's unknown how to establish a connection for the given type Arguments: - connection_name (str): Name of the connection, for instance, netmiko, paramiko, - napalm... + connection: Name of the connection, for instance, netmiko, paramiko, napalm... Returns: - An already established connection of type ``connection`` + An already established connection """ + if self.nornir: + config = self.nornir.config + else: + config = None if connection not in self.connections: - try: - conn_task = self.nornir.available_connections[connection] - except KeyError: - raise AttributeError( - "not sure how to establish a connection for {}".format(connection) - ) - - # We use `filter(name=self.name)` to call the connection task for only - # the given host. We also have to set `num_workers=1` because chances are - # we are already inside a thread - # Task should establish a connection and populate self.connection[connection] - r = self.nornir.filter(name=self.name).run(conn_task, num_workers=1) - if r[self.name].exception: - raise r[self.name].exception + self.open_connection( + connection, + self.host, + self.username, + self.password, + self.ssh_port, + self.network_api_port, + self.os, + self.nos, + self.get(f"{connection}_options", {}), + config, + ) + return self.connections[connection].connection + + def get_connection_state(self, connection: str) -> Dict[str, Any]: + """ + For an already established connection return its state. + """ + if connection not in self.connections: + raise ConnectionNotOpen(connection) + + return self.connections[connection].state + + def open_connection( + self, + connection: str, + hostname: Optional[str] = None, + username: Optional[str] = None, + password: Optional[str] = None, + ssh_port: Optional[int] = None, + network_api_port: Optional[int] = None, + operating_system: Optional[str] = None, + nos: Optional[str] = None, + connection_options: Optional[Dict[str, Any]] = None, + configuration: Optional[Config] = None, + default_to_host_attributes: bool = True, + ) -> None: + """ + Open a new connection. + + If ``default_to_host_attributes`` is set to ``True`` arguments will default to host + attributes if not specified. + Raises: + AttributeError: if it's unknown how to establish a connection for the given type + + Returns: + An already established connection + """ + if connection in self.connections: + raise ConnectionAlreadyOpen(connection) + + self.connections[connection] = self.nornir.get_connection_type(connection)() + if default_to_host_attributes: + self.connections[connection].open( + hostname=hostname if hostname is not None else self.host, + username=username if username is not None else self.username, + password=password if password is not None else self.password, + ssh_port=ssh_port if ssh_port is not None else self.ssh_port, + network_api_port=network_api_port + if network_api_port is not None + else self.network_api_port, + operating_system=operating_system + if operating_system is not None + else self.os, + nos=nos if nos is not None else self.nos, + connection_options=connection_options + if connection_options is not None + else self.get(f"{connection}_options"), + configuration=configuration + if configuration is not None + else self.nornir.config, + ) + else: + self.connections[connection].open( + hostname=hostname, + username=username, + password=password, + ssh_port=ssh_port, + network_api_port=network_api_port, + operating_system=operating_system, + nos=nos, + connection_options=connection_options, + configuration=configuration, + ) return self.connections[connection] + def close_connection(self, connection: str) -> None: + """ Close the connection""" + if connection not in self.connections: + raise ConnectionNotOpen(connection) + + self.connections.pop(connection).close() + + def close_connections(self) -> None: + for connection in self.connections: + self.close_connection(connection) + class Group(Host): """Same as :obj:`Host`""" diff --git a/nornir/plugins/connections/__init__.py b/nornir/plugins/connections/__init__.py new file mode 100644 index 00000000..5fc891fe --- /dev/null +++ b/nornir/plugins/connections/__init__.py @@ -0,0 +1,16 @@ +from typing import Dict, TYPE_CHECKING, Type + + +from .napalm import Napalm +from .netmiko import Netmiko +from .paramiko import Paramiko + +if TYPE_CHECKING: + from nornir.core.connections import ConnectionPlugin # noqa + + +available_connections: Dict[str, Type["ConnectionPlugin"]] = { + "napalm": Napalm, + "netmiko": Netmiko, + "paramiko": Paramiko, +} diff --git a/nornir/plugins/connections/napalm.py b/nornir/plugins/connections/napalm.py new file mode 100644 index 00000000..701adc92 --- /dev/null +++ b/nornir/plugins/connections/napalm.py @@ -0,0 +1,51 @@ +from typing import Any, Dict, Optional + +from napalm import get_network_driver + +from nornir.core.configuration import Config +from nornir.core.connections import ConnectionPlugin + + +class Napalm(ConnectionPlugin): + """ + This plugin connects to the device using the NAPALM driver and sets the + relevant connection. + + Inventory: + napalm_options: maps directly to ``optional_args`` when establishing the connection + nornir_network_api_port: maps to ``optional_args["port"]`` + napalm_options["timeout"]: maps to ``timeout``. + """ + + def open( + self, + hostname: str, + username: str, + password: str, + ssh_port: int, + network_api_port: int, + operating_system: str, + nos: str, + connection_options: Optional[Dict[str, Any]] = None, + configuration: Optional[Config] = None, + ) -> None: + connection_options = connection_options or {} + if network_api_port: + connection_options["port"] = network_api_port + + parameters = { + "hostname": hostname, + "username": username, + "password": password, + "optional_args": connection_options or {}, + } + if connection_options.get("timeout"): + parameters["timeout"] = connection_options["timeout"] + + network_driver = get_network_driver(nos) + connection = network_driver(**parameters) + connection.open() + self.connection = connection + + def close(self) -> None: + self.connection.close() diff --git a/nornir/plugins/connections/netmiko.py b/nornir/plugins/connections/netmiko.py new file mode 100644 index 00000000..37ebc1f3 --- /dev/null +++ b/nornir/plugins/connections/netmiko.py @@ -0,0 +1,56 @@ +from typing import Any, Dict, Optional + +from netmiko import ConnectHandler + +from nornir.core.configuration import Config +from nornir.core.connections import ConnectionPlugin + +napalm_to_netmiko_map = { + "ios": "cisco_ios", + "nxos": "cisco_nxos", + "eos": "arista_eos", + "junos": "juniper_junos", + "iosxr": "cisco_xr", +} + + +class Netmiko(ConnectionPlugin): + """ + This plugin connects to the device using the NAPALM driver and sets the + relevant connection. + + Inventory: + netmiko_options: maps to argument passed to ``ConnectHandler``. + nornir_network_ssh_port: maps to ``port`` + """ + + def open( + self, + hostname: str, + username: str, + password: str, + ssh_port: int, + network_api_port: int, + operating_system: str, + nos: str, + connection_options: Optional[Dict[str, Any]] = None, + configuration: Optional[Config] = None, + ) -> None: + parameters = { + "host": hostname, + "username": username, + "password": password, + "port": ssh_port, + } + + if nos is not None: + # Look device_type up in corresponding map, if no entry return the host.nos unmodified + device_type = napalm_to_netmiko_map.get(nos, nos) + parameters["device_type"] = device_type + + netmiko_connection_args = connection_options or {} + netmiko_connection_args.update(parameters) + self.connection = ConnectHandler(**netmiko_connection_args) + + def close(self) -> None: + self.connection.disconnect() diff --git a/nornir/plugins/connections/paramiko.py b/nornir/plugins/connections/paramiko.py new file mode 100644 index 00000000..c5034d36 --- /dev/null +++ b/nornir/plugins/connections/paramiko.py @@ -0,0 +1,71 @@ +import os +from typing import Any, Dict, Optional + +from nornir.core.configuration import Config +from nornir.core.connections import ConnectionPlugin + +import paramiko + + +class Paramiko(ConnectionPlugin): + """ + This plugin connects to the device with paramiko to the device and sets the + relevant connection. + + Inventory: + paramiko_options: maps to argument passed to ``ConnectHandler``. + nornir_network_ssh_port: maps to ``port`` + """ + + def open( + self, + hostname: str, + username: str, + password: str, + ssh_port: int, + network_api_port: int, + operating_system: str, + nos: str, + connection_options: Optional[Dict[str, Any]] = None, + configuration: Optional[Config] = None, + ) -> None: + connection_options = connection_options or {} + + client = paramiko.SSHClient() + client._policy = paramiko.WarningPolicy() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + ssh_config = paramiko.SSHConfig() + ssh_config_file = configuration.ssh_config_file # type: ignore + if os.path.exists(ssh_config_file): + with open(ssh_config_file) as f: + ssh_config.parse(f) + parameters = { + "hostname": hostname, + "username": username, + "password": password, + "port": ssh_port, + } + + user_config = ssh_config.lookup(hostname) + for k in ("hostname", "username", "port"): + if k in user_config: + parameters[k] = user_config[k] + + if "proxycommand" in user_config: + parameters["sock"] = paramiko.ProxyCommand(user_config["proxycommand"]) + + self.state["ssh_forward_agent"] = user_config.get("forwardagent") == "yes" + + # TODO configurable + # if ssh_key_file: + # parameters['key_filename'] = ssh_key_file + if "identityfile" in user_config: + parameters["key_filename"] = user_config["identityfile"] + + connection_options.update(parameters) + client.connect(**connection_options) + self.connection = client + + def close(self) -> None: + self.connection.close() diff --git a/nornir/plugins/tasks/commands/remote_command.py b/nornir/plugins/tasks/commands/remote_command.py index d6ee81ec..b803cb20 100644 --- a/nornir/plugins/tasks/commands/remote_command.py +++ b/nornir/plugins/tasks/commands/remote_command.py @@ -21,10 +21,11 @@ def remote_command(task: Task, command: str) -> Result: :obj:`nornir.core.exceptions.CommandError`: when there is a command error """ client = task.host.get_connection("paramiko") + connection_state = task.host.get_connection_state("paramiko") chan = client.get_transport().open_session() - if task.host._ssh_forward_agent: + if connection_state["ssh_forward_agent"]: AgentRequestHandler(chan) chan.exec_command(command) diff --git a/nornir/plugins/tasks/connections/__init__.py b/nornir/plugins/tasks/connections/__init__.py deleted file mode 100644 index b4a8240e..00000000 --- a/nornir/plugins/tasks/connections/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -from .napalm_connection import napalm_connection -from .netmiko_connection import netmiko_connection -from .paramiko_connection import paramiko_connection - - -available_connections = { - "napalm": napalm_connection, - "netmiko": netmiko_connection, - "paramiko": paramiko_connection, -} - -__all__ = ("napalm_connection", "netmiko_connection", "paramiko_connection") diff --git a/nornir/plugins/tasks/connections/napalm_connection.py b/nornir/plugins/tasks/connections/napalm_connection.py deleted file mode 100644 index 77bf0651..00000000 --- a/nornir/plugins/tasks/connections/napalm_connection.py +++ /dev/null @@ -1,59 +0,0 @@ -from typing import Optional, Dict, Any - -from napalm import get_network_driver - -from nornir.core.task import Task - - -def napalm_connection( - task: Task, timeout: int = 60, optional_args: Optional[Dict[str, Any]] = None -) -> None: - """ - This task connects to the device using the NAPALM driver and sets the - relevant connection. - - Arguments: - timeout: defaults to 60 - optional_args: defaults to `{"port": task.host["nornir_network_api_port"]}` - - Inventory: - napalm_options: maps directly to ``optional_args`` when establishing the connection - network_api_port: maps to ``optional_args["port"]`` - """ - host = task.host - - parameters = { - "hostname": host.host, - "username": host.username, - "password": host.password, - "timeout": timeout, - "optional_args": optional_args or host.get("napalm_options", {}), - } - - platform = host.nos - api_platforms = ["nxos", "eos", "iosxr", "junos"] - ssh_platforms = ["nxos_ssh", "ios"] - - # If port is set in optional_args that will control the port setting (else look to inventory) - if "port" not in parameters["optional_args"]: - if platform in api_platforms and host.network_api_port: - parameters["optional_args"]["port"] = host.network_api_port - elif platform in ssh_platforms and host.ssh_port: - parameters["optional_args"]["port"] = host.ssh_port - - # Setting host.nos to 'nxos' is potentially ambiguous - if platform == "nxos": - if not host.network_api_port: - if host.ssh_port or parameters["optional_args"].get("port") == 22: - platform = "nxos_ssh" - - # Fallback for community drivers (priority api_port over ssh_port) - if platform not in (api_platforms + ssh_platforms): - if host.network_api_port: - parameters["optional_args"]["port"] = host.network_api_port - elif host.ssh_port: - parameters["optional_args"]["port"] = host.ssh_port - - network_driver = get_network_driver(platform) - host.connections["napalm"] = network_driver(**parameters) - host.connections["napalm"].open() diff --git a/nornir/plugins/tasks/connections/netmiko_connection.py b/nornir/plugins/tasks/connections/netmiko_connection.py deleted file mode 100644 index 02f9e815..00000000 --- a/nornir/plugins/tasks/connections/netmiko_connection.py +++ /dev/null @@ -1,42 +0,0 @@ -from typing import Any - -from netmiko import ConnectHandler - -from nornir.core.task import Task - -napalm_to_netmiko_map = { - "ios": "cisco_ios", - "nxos": "cisco_nxos", - "eos": "arista_eos", - "junos": "juniper_junos", - "iosxr": "cisco_xr", -} - - -def netmiko_connection(task: Task, **netmiko_args: Any) -> None: - """Connect to the host using Netmiko and set the relevant connection in the connection map. - - Precedence: ``**netmiko_args`` > discrete inventory attributes > inventory netmiko_options - - Arguments: - ``**netmiko_args``: All supported Netmiko ConnectHandler arguments - """ - host = task.host - parameters = { - "host": host.host, - "username": host.username, - "password": host.password, - } - if host.ssh_port: - parameters["port"] = host.ssh_port - - if host.nos is not None: - # Look device_type up in corresponding map, if no entry return the host.nos unmodified - device_type = napalm_to_netmiko_map.get(host.nos, host.nos) - parameters["device_type"] = device_type - - # Precedence order: **netmiko_args > discrete inventory attributes > inventory netmiko_options - netmiko_connection_args = host.get("netmiko_options", {}) - netmiko_connection_args.update(parameters) - netmiko_connection_args.update(netmiko_args) - host.connections["netmiko"] = ConnectHandler(**netmiko_connection_args) diff --git a/nornir/plugins/tasks/connections/paramiko_connection.py b/nornir/plugins/tasks/connections/paramiko_connection.py deleted file mode 100644 index 5cb330c4..00000000 --- a/nornir/plugins/tasks/connections/paramiko_connection.py +++ /dev/null @@ -1,50 +0,0 @@ -import os - -import paramiko - -from nornir.core.task import Task - - -def paramiko_connection(task: Task) -> None: - """ - This tasks connects to the device with paramiko to the device and sets the - relevant connection. - """ - host = task.host - - client = paramiko.SSHClient() - client._policy = paramiko.WarningPolicy() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - - ssh_config = paramiko.SSHConfig() - ssh_config_file = task.nornir.config.ssh_config_file - if os.path.exists(ssh_config_file): - with open(ssh_config_file) as f: - ssh_config.parse(f) - - parameters = { - "hostname": host.host, - "username": host.username, - "password": host.password, - } - if host.ssh_port: - parameters["port"] = host.ssh_port - - user_config = ssh_config.lookup(host.host) - for k in ("hostname", "username", "port"): - if k in user_config: - parameters[k] = user_config[k] - - if "proxycommand" in user_config: - parameters["sock"] = paramiko.ProxyCommand(user_config["proxycommand"]) - - task.host._ssh_forward_agent = user_config.get("forwardagent") == "yes" - - # TODO configurable - # if ssh_key_file: - # parameters['key_filename'] = ssh_key_file - if "identityfile" in user_config: - parameters["key_filename"] = user_config["identityfile"] - - client.connect(**parameters) - host.connections["paramiko"] = client diff --git a/nornir/plugins/tasks/networking/napalm_get.py b/nornir/plugins/tasks/networking/napalm_get.py index 49a32c12..8fee7de8 100644 --- a/nornir/plugins/tasks/networking/napalm_get.py +++ b/nornir/plugins/tasks/networking/napalm_get.py @@ -1,5 +1,5 @@ import copy -from typing import List, Optional, Dict, Any +from typing import Any, Dict, List, Optional from nornir.core.task import Result, Task diff --git a/nornir/plugins/tasks/networking/napalm_validate.py b/nornir/plugins/tasks/networking/napalm_validate.py index 09234c69..2a747385 100644 --- a/nornir/plugins/tasks/networking/napalm_validate.py +++ b/nornir/plugins/tasks/networking/napalm_validate.py @@ -1,4 +1,5 @@ -from typing import Optional, Dict, Any +from typing import Any, Dict, Optional + from nornir.core.task import Result, Task ValidationSourceData = Optional[Dict[str, Dict[str, Any]]] diff --git a/nornir/plugins/tasks/networking/netmiko_file_transfer.py b/nornir/plugins/tasks/networking/netmiko_file_transfer.py index ee40e2ee..b47c7950 100644 --- a/nornir/plugins/tasks/networking/netmiko_file_transfer.py +++ b/nornir/plugins/tasks/networking/netmiko_file_transfer.py @@ -1,8 +1,9 @@ from typing import Any -from nornir.core.task import Result, Task from netmiko import file_transfer +from nornir.core.task import Result, Task + def netmiko_file_transfer( task: Task, source_file: str, dest_file: str, **kwargs: Any diff --git a/nornir/plugins/tasks/networking/netmiko_send_config.py b/nornir/plugins/tasks/networking/netmiko_send_config.py index 49f3eb48..56633f69 100644 --- a/nornir/plugins/tasks/networking/netmiko_send_config.py +++ b/nornir/plugins/tasks/networking/netmiko_send_config.py @@ -1,4 +1,5 @@ -from typing import Optional, Any, List +from typing import Any, List, Optional + from nornir.core.task import Result, Task diff --git a/setup.cfg b/setup.cfg index 945dd6ff..e9a03184 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,15 @@ warn_redundant_casts = True warn_unused_configs = True ignore_missing_imports = True +[mypy-nornir.core.connections] +check_untyped_defs = True +disallow_any_generics = True +# Turn on the next flag once the whole codebase is annotated (Phase 2) +# disallow_untyped_calls = True +strict_optional = True +warn_unused_ignores = True +ignore_errors = False + [mypy-nornir.plugins.*] check_untyped_defs = True disallow_any_generics = True diff --git a/setup.py b/setup.py index 80354a5e..0055495c 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ __author__ = "dbarrosop@dravetech.com" __license__ = "Apache License, version 2" -__version__ = "1.1.0" +__version__ = "2.0.0" setup( name="nornir", diff --git a/tests/core/test_connections.py b/tests/core/test_connections.py new file mode 100644 index 00000000..e6d04097 --- /dev/null +++ b/tests/core/test_connections.py @@ -0,0 +1,96 @@ +from typing import Any, Dict, Optional + +from nornir.core.configuration import Config +from nornir.core.connections import ConnectionPlugin +from nornir.core.exceptions import ConnectionAlreadyOpen, ConnectionNotOpen + + +class DummyConnectionPlugin(ConnectionPlugin): + """ + This plugin connects to the device using the NAPALM driver and sets the + relevant connection. + + Inventory: + napalm_options: maps directly to ``optional_args`` when establishing the connection + nornir_network_api_port: maps to ``optional_args["port"]`` + napalm_options["timeout"]: maps to ``timeout``. + """ + + def open( + self, + hostname: str, + username: str, + password: str, + ssh_port: int, + network_api_port: int, + operating_system: str, + nos: str, + connection_options: Optional[Dict[str, Any]] = None, + configuration: Optional[Config] = None, + ) -> None: + self.connection = True + self.state["something"] = "something" + + def close(self) -> None: + self.connection = False + + +def open_and_close_connection(task): + task.host.open_connection("dummy") + assert "dummy" in task.host.connections + task.host.close_connection("dummy") + assert "dummy" not in task.host.connections + + +def open_connection_twice(task): + task.host.open_connection("dummy") + assert "dummy" in task.host.connections + try: + task.host.open_connection("dummy") + raise Exception("I shouldn't make it here") + except ConnectionAlreadyOpen: + task.host.close_connection("dummy") + assert "dummy" not in task.host.connections + + +def close_not_opened_connection(task): + assert "dummy" not in task.host.connections + try: + task.host.close_connection("dummy") + raise Exception("I shouldn't make it here") + except ConnectionNotOpen: + assert "dummy" not in task.host.connections + + +def a_task(task): + task.host.get_connection("dummy") + + +class Test(object): + def test_open_and_close_connection(self, nornir): + nornir.data.available_connections["dummy"] = DummyConnectionPlugin + nr = nornir.filter(name="dev2.group_1") + r = nr.run(task=open_and_close_connection, num_workers=1) + assert len(r) == 1 + assert not r.failed + + def test_open_connection_twice(self, nornir): + nornir.data.available_connections["dummy"] = DummyConnectionPlugin + nr = nornir.filter(name="dev2.group_1") + r = nr.run(task=open_connection_twice, num_workers=1) + assert len(r) == 1 + assert not r.failed + + def test_close_not_opened_connection(self, nornir): + nornir.data.available_connections["dummy"] = DummyConnectionPlugin + nr = nornir.filter(name="dev2.group_1") + r = nr.run(task=close_not_opened_connection, num_workers=1) + assert len(r) == 1 + assert not r.failed + + def test_context_manager(self, nornir): + with nornir.filter(name="dev2.group_1") as nr: + nr.run(task=a_task) + assert "dummy" in nr.inventory.hosts["dev2.group_1"].connections + assert "dummy" not in nr.inventory.hosts["dev2.group_1"].connections + nornir.data.reset_failed_hosts() diff --git a/tests/plugins/tasks/networking/test_napalm_cli.py b/tests/plugins/tasks/networking/test_napalm_cli.py index c291a7a1..7cd66c52 100644 --- a/tests/plugins/tasks/networking/test_napalm_cli.py +++ b/tests/plugins/tasks/networking/test_napalm_cli.py @@ -1,7 +1,7 @@ import os # from nornir.core.exceptions import NornirExecutionError -from nornir.plugins.tasks import connections, networking +from nornir.plugins.tasks import networking # from napalm.base import exceptions @@ -11,11 +11,24 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__)) + "/mocked/napalm_cli" +def connect(task, connection_options): + if "napalm" in task.host.connections: + task.host.close_connection("napalm") + task.host.open_connection( + "napalm", + hostname=task.host.username, + password=task.host.password, + network_api_port=task.host.network_api_port, + nos=task.host.nos, + connection_options=connection_options, + ) + + class Test(object): def test_napalm_cli(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_cli"} d = nornir.filter(name="dev3.group_2") - d.run(connections.napalm_connection, optional_args=opt) + d.run(connect, connection_options=opt) result = d.run( networking.napalm_cli, commands=["show version", "show interfaces"] ) diff --git a/tests/plugins/tasks/networking/test_napalm_configure.py b/tests/plugins/tasks/networking/test_napalm_configure.py index 75391d13..144bffa5 100644 --- a/tests/plugins/tasks/networking/test_napalm_configure.py +++ b/tests/plugins/tasks/networking/test_napalm_configure.py @@ -1,19 +1,32 @@ import os -from nornir.plugins.tasks import connections, networking - from napalm.base import exceptions +from nornir.plugins.tasks import networking + THIS_DIR = os.path.dirname(os.path.realpath(__file__)) + "/mocked/napalm_configure" +def connect(task, connection_options): + if "napalm" in task.host.connections: + task.host.close_connection("napalm") + task.host.open_connection( + "napalm", + hostname=task.host.username, + password=task.host.password, + network_api_port=task.host.network_api_port, + nos=task.host.nos, + connection_options=connection_options, + ) + + class Test(object): def test_napalm_configure_change_dry_run(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_configure_change_dry_run"} configuration = "hostname changed-hostname" d = nornir.filter(name="dev3.group_2") - d.run(connections.napalm_connection, optional_args=opt) + d.run(connect, connection_options=opt) result = d.run(networking.napalm_configure, configuration=configuration) assert result for h, r in result.items(): @@ -24,7 +37,7 @@ def test_napalm_configure_change_commit(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_configure_change_commit/step1"} configuration = "hostname changed-hostname" d = nornir.filter(name="dev3.group_2") - d.run(connections.napalm_connection, optional_args=opt) + d.run(connect, connection_options=opt) result = d.run( networking.napalm_configure, dry_run=False, configuration=configuration ) @@ -33,7 +46,7 @@ def test_napalm_configure_change_commit(self, nornir): assert "+hostname changed-hostname" in r.diff assert r.changed opt = {"path": THIS_DIR + "/test_napalm_configure_change_commit/step2"} - d.run(connections.napalm_connection, optional_args=opt) + d.run(connect, connection_options=opt) result = d.run( networking.napalm_configure, dry_run=True, configuration=configuration ) @@ -47,7 +60,7 @@ def test_napalm_configure_change_error(self, nornir): configuration = "hostname changed_hostname" d = nornir.filter(name="dev3.group_2") - d.run(connections.napalm_connection, optional_args=opt) + d.run(connect, connection_options=opt) results = d.run(networking.napalm_configure, configuration=configuration) processed = False for result in results.values(): diff --git a/tests/plugins/tasks/networking/test_napalm_get.py b/tests/plugins/tasks/networking/test_napalm_get.py index f6b3003e..df9ede0c 100644 --- a/tests/plugins/tasks/networking/test_napalm_get.py +++ b/tests/plugins/tasks/networking/test_napalm_get.py @@ -1,16 +1,29 @@ import os -from nornir.plugins.tasks import connections, networking +from nornir.plugins.tasks import networking THIS_DIR = os.path.dirname(os.path.realpath(__file__)) + "/mocked/napalm_get" +def connect(task, connection_options): + if "napalm" in task.host.connections: + task.host.close_connection("napalm") + task.host.open_connection( + "napalm", + hostname=task.host.host, + password=task.host.password, + network_api_port=task.host.network_api_port, + nos=task.host.nos, + connection_options=connection_options, + ) + + class Test(object): def test_napalm_getters(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters"} d = nornir.filter(name="dev3.group_2") - d.run(connections.napalm_connection, optional_args=opt) + d.run(task=connect, connection_options=opt) result = d.run(networking.napalm_get, getters=["facts", "interfaces"]) assert result for h, r in result.items(): @@ -20,7 +33,7 @@ def test_napalm_getters(self, nornir): def test_napalm_getters_error(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_error"} d = nornir.filter(name="dev3.group_2") - d.run(connections.napalm_connection, optional_args=opt) + d.run(task=connect, connection_options=opt) results = d.run(networking.napalm_get, getters=["facts", "interfaces"]) processed = False @@ -33,7 +46,7 @@ def test_napalm_getters_error(self, nornir): def test_napalm_getters_with_options_error(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_single_with_options"} d = nornir.filter(name="dev3.group_2") - d.run(connections.napalm_connection, optional_args=opt) + d.run(task=connect, connection_options=opt) result = d.run( task=networking.napalm_get, getters=["config"], nonexistent="asdsa" ) @@ -46,7 +59,7 @@ def test_napalm_getters_with_options_error(self, nornir): def test_napalm_getters_with_options_error_optional_args(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_single_with_options"} d = nornir.filter(name="dev3.group_2") - d.run(connections.napalm_connection, optional_args=opt) + d.run(task=connect, connection_options=opt) result = d.run( task=networking.napalm_get, getters=["config"], @@ -61,7 +74,7 @@ def test_napalm_getters_with_options_error_optional_args(self, nornir): def test_napalm_getters_single_with_options(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_single_with_options"} d = nornir.filter(name="dev3.group_2") - d.run(connections.napalm_connection, optional_args=opt) + d.run(task=connect, connection_options=opt) result = d.run( task=networking.napalm_get, getters=["config"], retrieve="candidate" ) @@ -73,7 +86,7 @@ def test_napalm_getters_single_with_options(self, nornir): def test_napalm_getters_multiple_with_options(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_multiple_with_options"} d = nornir.filter(name="dev3.group_2") - d.run(connections.napalm_connection, optional_args=opt) + d.run(task=connect, connection_options=opt) result = d.run( task=networking.napalm_get, getters=["config", "facts"], diff --git a/tests/plugins/tasks/networking/test_napalm_validate.py b/tests/plugins/tasks/networking/test_napalm_validate.py index ac1ae347..fc34fd40 100644 --- a/tests/plugins/tasks/networking/test_napalm_validate.py +++ b/tests/plugins/tasks/networking/test_napalm_validate.py @@ -1,16 +1,29 @@ import os -from nornir.plugins.tasks import connections, networking +from nornir.plugins.tasks import networking THIS_DIR = os.path.dirname(os.path.realpath(__file__)) +def connect(task, connection_options): + if "napalm" in task.host.connections: + task.host.close_connection("napalm") + task.host.open_connection( + "napalm", + hostname=task.host.username, + password=task.host.password, + network_api_port=task.host.network_api_port, + nos=task.host.nos, + connection_options=connection_options, + ) + + class Test(object): def test_napalm_validate_src_ok(self, nornir): opt = {"path": THIS_DIR + "/mocked/napalm_get/test_napalm_getters"} d = nornir.filter(name="dev3.group_2") - d.run(connections.napalm_connection, optional_args=opt) + d.run(connect, connection_options=opt) result = d.run( networking.napalm_validate, src=THIS_DIR + "/data/validate_ok.yaml" ) @@ -21,7 +34,7 @@ def test_napalm_validate_src_ok(self, nornir): def test_napalm_validate_src_error(self, nornir): opt = {"path": THIS_DIR + "/mocked/napalm_get/test_napalm_getters"} d = nornir.filter(name="dev3.group_2") - d.run(connections.napalm_connection, optional_args=opt) + d.run(connect, connection_options=opt) result = d.run( networking.napalm_validate, src=THIS_DIR + "/data/validate_error.yaml" @@ -34,7 +47,7 @@ def test_napalm_validate_src_error(self, nornir): def test_napalm_validate_src_validate_source(self, nornir): opt = {"path": THIS_DIR + "/mocked/napalm_get/test_napalm_getters"} d = nornir.filter(name="dev3.group_2") - d.run(connections.napalm_connection, optional_args=opt) + d.run(connect, connection_options=opt) validation_dict = [{"get_interfaces": {"Ethernet1": {"description": ""}}}] diff --git a/tests/plugins/tasks/networking/test_netmiko_send_command.py b/tests/plugins/tasks/networking/test_netmiko_send_command.py index d7bf826c..bf38dae4 100644 --- a/tests/plugins/tasks/networking/test_netmiko_send_command.py +++ b/tests/plugins/tasks/networking/test_netmiko_send_command.py @@ -1,16 +1,7 @@ -from nornir.plugins.tasks import connections, networking +from nornir.plugins.tasks import networking class Test(object): - def test_explicit_netmiko_connection(self, nornir): - nornir.filter(name="dev4.group_2").run(task=connections.netmiko_connection) - result = nornir.filter(name="dev4.group_2").run( - networking.netmiko_send_command, command_string="hostname" - ) - assert result - for h, r in result.items(): - assert h == r.result.strip() - def test_netmiko_send_command(self, nornir): result = nornir.filter(name="dev4.group_2").run( networking.netmiko_send_command, command_string="hostname" diff --git a/tests/plugins/tasks/networking/test_netmiko_send_config.py b/tests/plugins/tasks/networking/test_netmiko_send_config.py index 3558888c..5ad1b783 100644 --- a/tests/plugins/tasks/networking/test_netmiko_send_config.py +++ b/tests/plugins/tasks/networking/test_netmiko_send_config.py @@ -1,16 +1,7 @@ -from nornir.plugins.tasks import connections, networking +from nornir.plugins.tasks import networking class Test(object): - def test_explicit_netmiko_connection(self, nornir): - nornir.filter(name="dev4.group_2").run(task=connections.netmiko_connection) - result = nornir.filter(name="dev4.group_2").run( - networking.netmiko_send_config, config_commands="hostname" - ) - assert result - for h, r in result.items(): - assert h in r.result.strip() - def test_netmiko_send_command(self, nornir): result = nornir.filter(name="dev4.group_2").run( networking.netmiko_send_config, config_commands="hostname" From b5a49a783d6c840ec057628184a8f0feac166f01 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Fri, 3 Aug 2018 13:41:48 +0200 Subject: [PATCH 017/109] update black --- nornir/plugins/inventory/netbox.py | 1 - tests/plugins/inventory/test_netbox.py | 1 - 2 files changed, 2 deletions(-) diff --git a/nornir/plugins/inventory/netbox.py b/nornir/plugins/inventory/netbox.py index 6b95d8a8..133875de 100644 --- a/nornir/plugins/inventory/netbox.py +++ b/nornir/plugins/inventory/netbox.py @@ -7,7 +7,6 @@ class NBInventory(Inventory): - def __init__( self, nb_url=None, diff --git a/tests/plugins/inventory/test_netbox.py b/tests/plugins/inventory/test_netbox.py index c48ba17b..8e632b43 100644 --- a/tests/plugins/inventory/test_netbox.py +++ b/tests/plugins/inventory/test_netbox.py @@ -26,7 +26,6 @@ def transform_function(host): class Test(object): - def test_inventory(self, requests_mock): inv = get_inv(requests_mock, "2.3.5") with open("{}/{}/expected.json".format(BASE_PATH, "2.3.5"), "r") as f: From 79746524004faffb203f0c3d501aac0ec3fbc4bc Mon Sep 17 00:00:00 2001 From: David Barroso Date: Mon, 6 Aug 2018 00:13:19 +0200 Subject: [PATCH 018/109] consolidate inventory attributes --- nornir/core/connections.py | 6 +- nornir/core/inventory.py | 108 +++++++++--------- nornir/plugins/connections/napalm.py | 12 +- nornir/plugins/connections/netmiko.py | 12 +- nornir/plugins/connections/paramiko.py | 8 +- nornir/plugins/tasks/networking/tcp_ping.py | 2 +- tests/core/test_connections.py | 7 +- tests/core/test_filter.py | 2 +- tests/core/test_inventory.py | 11 +- tests/inventory_data/groups.yaml | 8 +- tests/inventory_data/hosts.yaml | 27 ++--- .../tasks/networking/test_napalm_cli.py | 7 +- .../tasks/networking/test_napalm_configure.py | 7 +- .../tasks/networking/test_napalm_get.py | 7 +- .../tasks/networking/test_napalm_validate.py | 7 +- 15 files changed, 99 insertions(+), 132 deletions(-) diff --git a/nornir/core/connections.py b/nornir/core/connections.py index 0763512b..078a9eea 100644 --- a/nornir/core/connections.py +++ b/nornir/core/connections.py @@ -28,10 +28,8 @@ def open( hostname: str, username: str, password: str, - ssh_port: int, - network_api_port: int, - operating_system: str, - nos: str, + port: int, + device_type: str, connection_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index d1f929d7..49209a31 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -196,42 +196,52 @@ def nornir(self, value): self._nornir = value @property - def host(self): - """String used to connect to the device. Either ``nornir_host`` or ``self.name``""" - return self.get("nornir_host", self.name) + def hostname(self): + """String used to connect to the device. Either ``hostname`` or ``self.name``""" + return self.get("hostname", self.name) + + @property + def port(self): + """Either ``port`` or ``None``.""" + return self.get("port") @property def username(self): """Either ``nornir_username`` or user running the script.""" - return self.get("nornir_username", getpass.getuser()) + return self.get("username", getpass.getuser()) @property def password(self): """Either ``nornir_password`` or empty string.""" - return self.get("nornir_password", "") - - @property - def ssh_port(self): - """Either ``nornir_ssh_port`` or ``None``.""" - return self.get("nornir_ssh_port") + return self.get("password", "") @property - def network_api_port(self): - """ - For network equipment this is the port where the device's API is listening to. - Either ``nornir_network_api_port`` or ``None``. - """ - return self.get("nornir_network_api_port") - - @property - def os(self): - """OS the device is running. Defaults to ``nornir_os``.""" - return self.get("nornir_os") - - @property - def nos(self): - """Network OS the device is running. Defaults to ``nornir_nos``.""" - return self.get("nornir_nos") + def device_type(self): + """OS the device is running. Defaults to ``device_type``.""" + return self.get("device_type") + + def get_connection_parameters( + self, connection: Optional[str] = None + ) -> Dict[str, Any]: + if not connection: + return { + "hostname": self.hostname, + "port": self.port, + "username": self.username, + "password": self.password, + "device_type": self.device_type, + "connection_options": {}, + } + else: + conn_params = self.get("connection_options", {}).get(connection, {}) + return { + "hostname": conn_params.get("hostname", self.hostname), + "port": conn_params.get("port", self.port), + "username": conn_params.get("username", self.username), + "password": conn_params.get("password", self.password), + "device_type": conn_params.get("device_type", self.device_type), + "connection_options": conn_params.get("additional_options", {}), + } def get_connection(self, connection: str) -> Any: """ @@ -257,15 +267,8 @@ def get_connection(self, connection: str) -> Any: if connection not in self.connections: self.open_connection( connection, - self.host, - self.username, - self.password, - self.ssh_port, - self.network_api_port, - self.os, - self.nos, - self.get(f"{connection}_options", {}), - config, + **self.get_connection_parameters(connection), + configuration=config, ) return self.connections[connection].connection @@ -284,11 +287,9 @@ def open_connection( hostname: Optional[str] = None, username: Optional[str] = None, password: Optional[str] = None, - ssh_port: Optional[int] = None, - network_api_port: Optional[int] = None, - operating_system: Optional[str] = None, - nos: Optional[str] = None, - connection_options: Optional[Dict[str, Any]] = None, + port: Optional[int] = None, + device_type: Optional[int] = None, + connection_options: Optional[int] = None, configuration: Optional[Config] = None, default_to_host_attributes: bool = True, ) -> None: @@ -309,21 +310,18 @@ def open_connection( self.connections[connection] = self.nornir.get_connection_type(connection)() if default_to_host_attributes: + conn_params = self.get_connection_parameters(connection) self.connections[connection].open( - hostname=hostname if hostname is not None else self.host, - username=username if username is not None else self.username, - password=password if password is not None else self.password, - ssh_port=ssh_port if ssh_port is not None else self.ssh_port, - network_api_port=network_api_port - if network_api_port is not None - else self.network_api_port, - operating_system=operating_system - if operating_system is not None - else self.os, - nos=nos if nos is not None else self.nos, + hostname=hostname if hostname is not None else conn_params["hostname"], + username=username if username is not None else conn_params["username"], + password=password if password is not None else conn_params["password"], + port=port if port is not None else conn_params["port"], + device_type=device_type + if device_type is not None + else conn_params["device_type"], connection_options=connection_options if connection_options is not None - else self.get(f"{connection}_options"), + else conn_params["connection_options"], configuration=configuration if configuration is not None else self.nornir.config, @@ -333,10 +331,8 @@ def open_connection( hostname=hostname, username=username, password=password, - ssh_port=ssh_port, - network_api_port=network_api_port, - operating_system=operating_system, - nos=nos, + port=port, + device_type=device_type, connection_options=connection_options, configuration=configuration, ) diff --git a/nornir/plugins/connections/napalm.py b/nornir/plugins/connections/napalm.py index 701adc92..b018ebac 100644 --- a/nornir/plugins/connections/napalm.py +++ b/nornir/plugins/connections/napalm.py @@ -22,16 +22,14 @@ def open( hostname: str, username: str, password: str, - ssh_port: int, - network_api_port: int, - operating_system: str, - nos: str, + port: int, + device_type: str, connection_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: connection_options = connection_options or {} - if network_api_port: - connection_options["port"] = network_api_port + if port: + connection_options["port"] = port parameters = { "hostname": hostname, @@ -42,7 +40,7 @@ def open( if connection_options.get("timeout"): parameters["timeout"] = connection_options["timeout"] - network_driver = get_network_driver(nos) + network_driver = get_network_driver(device_type) connection = network_driver(**parameters) connection.open() self.connection = connection diff --git a/nornir/plugins/connections/netmiko.py b/nornir/plugins/connections/netmiko.py index 37ebc1f3..6ae3bed0 100644 --- a/nornir/plugins/connections/netmiko.py +++ b/nornir/plugins/connections/netmiko.py @@ -29,10 +29,8 @@ def open( hostname: str, username: str, password: str, - ssh_port: int, - network_api_port: int, - operating_system: str, - nos: str, + port: int, + device_type: str, connection_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: @@ -40,12 +38,12 @@ def open( "host": hostname, "username": username, "password": password, - "port": ssh_port, + "port": port, } - if nos is not None: + if device_type is not None: # Look device_type up in corresponding map, if no entry return the host.nos unmodified - device_type = napalm_to_netmiko_map.get(nos, nos) + device_type = napalm_to_netmiko_map.get(device_type, device_type) parameters["device_type"] = device_type netmiko_connection_args = connection_options or {} diff --git a/nornir/plugins/connections/paramiko.py b/nornir/plugins/connections/paramiko.py index c5034d36..abb4b521 100644 --- a/nornir/plugins/connections/paramiko.py +++ b/nornir/plugins/connections/paramiko.py @@ -22,10 +22,8 @@ def open( hostname: str, username: str, password: str, - ssh_port: int, - network_api_port: int, - operating_system: str, - nos: str, + port: int, + device_type: str, connection_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: @@ -44,7 +42,7 @@ def open( "hostname": hostname, "username": username, "password": password, - "port": ssh_port, + "port": port, } user_config = ssh_config.lookup(hostname) diff --git a/nornir/plugins/tasks/networking/tcp_ping.py b/nornir/plugins/tasks/networking/tcp_ping.py index 85ab248f..be254845 100644 --- a/nornir/plugins/tasks/networking/tcp_ping.py +++ b/nornir/plugins/tasks/networking/tcp_ping.py @@ -32,7 +32,7 @@ def tcp_ping( else: raise ValueError("Invalid value for 'ports'") - host = host or task.host.host + host = host or task.host.hostname result = {} for port in ports: diff --git a/tests/core/test_connections.py b/tests/core/test_connections.py index e6d04097..f7f59fbf 100644 --- a/tests/core/test_connections.py +++ b/tests/core/test_connections.py @@ -21,10 +21,8 @@ def open( hostname: str, username: str, password: str, - ssh_port: int, - network_api_port: int, - operating_system: str, - nos: str, + port: int, + device_type: str, connection_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: @@ -89,6 +87,7 @@ def test_close_not_opened_connection(self, nornir): assert not r.failed def test_context_manager(self, nornir): + nornir.data.available_connections["dummy"] = DummyConnectionPlugin with nornir.filter(name="dev2.group_1") as nr: nr.run(task=a_task) assert "dummy" in nr.inventory.hosts["dev2.group_1"].connections diff --git a/tests/core/test_filter.py b/tests/core/test_filter.py index 3555099e..adaa8fdf 100644 --- a/tests/core/test_filter.py +++ b/tests/core/test_filter.py @@ -113,7 +113,7 @@ def test_filtering_by_attribute_name(self): assert filtered == ["dev1.group_1"] def test_filtering_string_in_list(self): - f = F(nornir_nos__in=["linux", "mock"]) + f = F(device_type__in=["linux", "mock"]) filtered = sorted(list((inventory.filter(f).hosts.keys()))) assert filtered == ["dev3.group_2", "dev4.group_2"] diff --git a/tests/core/test_inventory.py b/tests/core/test_inventory.py index e4ab5e12..c68aaa4a 100644 --- a/tests/core/test_inventory.py +++ b/tests/core/test_inventory.py @@ -222,18 +222,17 @@ def test_to_dict(self): "my_var": "comes_from_dev1.group_1", "www_server": "nginx", "role": "www", - "nornir_ssh_port": 65001, - "nornir_nos": "eos", + "port": 65001, + "device_type": "eos", }, "dev3.group_2": { "name": "dev3.group_2", "groups": ["group_2"], "www_server": "apache", "role": "www", - "nornir_ssh_port": 65003, - "nornir_network_api_port": 12443, - "nornir_os": "linux", - "nornir_nos": "mock", + "port": 65003, + "device_type": "linux", + "connection_options": {"napalm": {"device_type": "mock"}}, }, }, "groups": { diff --git a/tests/inventory_data/groups.yaml b/tests/inventory_data/groups.yaml index b05fc9ed..d094c207 100644 --- a/tests/inventory_data/groups.yaml +++ b/tests/inventory_data/groups.yaml @@ -1,10 +1,10 @@ --- defaults: my_var: comes_from_defaults - nornir_host: 127.0.0.1 - nornir_username: root - nornir_password: docker - nornir_os: linux + hostname: 127.0.0.1 + username: root + password: docker + device_type: linux parent_group: a_var: blah diff --git a/tests/inventory_data/hosts.yaml b/tests/inventory_data/hosts.yaml index 329d4b57..e3cd99db 100644 --- a/tests/inventory_data/hosts.yaml +++ b/tests/inventory_data/hosts.yaml @@ -5,8 +5,8 @@ dev1.group_1: my_var: comes_from_dev1.group_1 www_server: nginx role: www - nornir_ssh_port: 65001 - nornir_nos: eos + port: 65001 + device_type: eos nested_data: a_dict: a: 1 @@ -18,8 +18,8 @@ dev2.group_1: groups: - group_1 role: db - nornir_ssh_port: 65002 - nornir_nos: junos + port: 65002 + device_type: junos nested_data: a_dict: b: 2 @@ -32,18 +32,19 @@ dev3.group_2: - group_2 www_server: apache role: www - nornir_ssh_port: 65003 - nornir_network_api_port: 12443 - nornir_os: linux - nornir_nos: mock - # nornir_username: vagrant - # nornir_password: vagrant + port: 65003 + device_type: linux + connection_options: + napalm: + device_type: mock dev4.group_2: groups: - group_2 my_var: comes_from_dev4.group_2 role: db - nornir_ssh_port: 65004 - nornir_network_api_port: 65004 - nornir_nos: linux + port: 65004 + device_type: linux + connection_options: + napalm: + device_type: mock diff --git a/tests/plugins/tasks/networking/test_napalm_cli.py b/tests/plugins/tasks/networking/test_napalm_cli.py index 7cd66c52..3236636c 100644 --- a/tests/plugins/tasks/networking/test_napalm_cli.py +++ b/tests/plugins/tasks/networking/test_napalm_cli.py @@ -15,12 +15,7 @@ def connect(task, connection_options): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", - hostname=task.host.username, - password=task.host.password, - network_api_port=task.host.network_api_port, - nos=task.host.nos, - connection_options=connection_options, + "napalm", connection_options=connection_options, default_to_host_attributes=True ) diff --git a/tests/plugins/tasks/networking/test_napalm_configure.py b/tests/plugins/tasks/networking/test_napalm_configure.py index 144bffa5..45938cd3 100644 --- a/tests/plugins/tasks/networking/test_napalm_configure.py +++ b/tests/plugins/tasks/networking/test_napalm_configure.py @@ -12,12 +12,7 @@ def connect(task, connection_options): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", - hostname=task.host.username, - password=task.host.password, - network_api_port=task.host.network_api_port, - nos=task.host.nos, - connection_options=connection_options, + "napalm", connection_options=connection_options, default_to_host_attributes=True ) diff --git a/tests/plugins/tasks/networking/test_napalm_get.py b/tests/plugins/tasks/networking/test_napalm_get.py index df9ede0c..0a7556a3 100644 --- a/tests/plugins/tasks/networking/test_napalm_get.py +++ b/tests/plugins/tasks/networking/test_napalm_get.py @@ -10,12 +10,7 @@ def connect(task, connection_options): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", - hostname=task.host.host, - password=task.host.password, - network_api_port=task.host.network_api_port, - nos=task.host.nos, - connection_options=connection_options, + "napalm", connection_options=connection_options, default_to_host_attributes=True ) diff --git a/tests/plugins/tasks/networking/test_napalm_validate.py b/tests/plugins/tasks/networking/test_napalm_validate.py index fc34fd40..6ee79742 100644 --- a/tests/plugins/tasks/networking/test_napalm_validate.py +++ b/tests/plugins/tasks/networking/test_napalm_validate.py @@ -10,12 +10,7 @@ def connect(task, connection_options): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", - hostname=task.host.username, - password=task.host.password, - network_api_port=task.host.network_api_port, - nos=task.host.nos, - connection_options=connection_options, + "napalm", connection_options=connection_options, default_to_host_attributes=True ) From 7d9945f0cf042f9d1e0e946c1c394d00425b13d8 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Mon, 6 Aug 2018 20:53:44 +0200 Subject: [PATCH 019/109] device_type is now platform --- nornir/core/connections.py | 2 +- nornir/core/inventory.py | 20 +++++++++---------- nornir/plugins/connections/napalm.py | 4 ++-- nornir/plugins/connections/netmiko.py | 10 +++++----- nornir/plugins/connections/paramiko.py | 2 +- nornir/plugins/inventory/ansible.py | 8 ++++---- nornir/plugins/inventory/netbox.py | 6 +++--- tests/core/test_connections.py | 2 +- tests/core/test_filter.py | 2 +- tests/core/test_inventory.py | 6 +++--- tests/inventory_data/groups.yaml | 2 +- tests/inventory_data/hosts.yaml | 12 +++++------ .../inventory/ansible/ini/expected/hosts.yaml | 4 ++-- .../ansible/yaml/expected/hosts.yaml | 4 ++-- .../ansible/yaml2/expected/hosts.yaml | 4 ++-- .../inventory/netbox/2.3.5/expected.json | 12 +++++------ .../2.3.5/expected_transform_function.json | 12 +++++------ tests/plugins/inventory/test_netbox.py | 2 +- 18 files changed, 57 insertions(+), 57 deletions(-) diff --git a/nornir/core/connections.py b/nornir/core/connections.py index 078a9eea..1c03b39c 100644 --- a/nornir/core/connections.py +++ b/nornir/core/connections.py @@ -29,7 +29,7 @@ def open( username: str, password: str, port: int, - device_type: str, + platform: str, connection_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 49209a31..7731526d 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -216,9 +216,9 @@ def password(self): return self.get("password", "") @property - def device_type(self): - """OS the device is running. Defaults to ``device_type``.""" - return self.get("device_type") + def platform(self): + """OS the device is running. Defaults to ``platform``.""" + return self.get("platform") def get_connection_parameters( self, connection: Optional[str] = None @@ -229,7 +229,7 @@ def get_connection_parameters( "port": self.port, "username": self.username, "password": self.password, - "device_type": self.device_type, + "platform": self.platform, "connection_options": {}, } else: @@ -239,7 +239,7 @@ def get_connection_parameters( "port": conn_params.get("port", self.port), "username": conn_params.get("username", self.username), "password": conn_params.get("password", self.password), - "device_type": conn_params.get("device_type", self.device_type), + "platform": conn_params.get("platform", self.platform), "connection_options": conn_params.get("additional_options", {}), } @@ -288,7 +288,7 @@ def open_connection( username: Optional[str] = None, password: Optional[str] = None, port: Optional[int] = None, - device_type: Optional[int] = None, + platform: Optional[int] = None, connection_options: Optional[int] = None, configuration: Optional[Config] = None, default_to_host_attributes: bool = True, @@ -316,9 +316,9 @@ def open_connection( username=username if username is not None else conn_params["username"], password=password if password is not None else conn_params["password"], port=port if port is not None else conn_params["port"], - device_type=device_type - if device_type is not None - else conn_params["device_type"], + platform=platform + if platform is not None + else conn_params["platform"], connection_options=connection_options if connection_options is not None else conn_params["connection_options"], @@ -332,7 +332,7 @@ def open_connection( username=username, password=password, port=port, - device_type=device_type, + platform=platform, connection_options=connection_options, configuration=configuration, ) diff --git a/nornir/plugins/connections/napalm.py b/nornir/plugins/connections/napalm.py index b018ebac..e907b395 100644 --- a/nornir/plugins/connections/napalm.py +++ b/nornir/plugins/connections/napalm.py @@ -23,7 +23,7 @@ def open( username: str, password: str, port: int, - device_type: str, + platform: str, connection_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: @@ -40,7 +40,7 @@ def open( if connection_options.get("timeout"): parameters["timeout"] = connection_options["timeout"] - network_driver = get_network_driver(device_type) + network_driver = get_network_driver(platform) connection = network_driver(**parameters) connection.open() self.connection = connection diff --git a/nornir/plugins/connections/netmiko.py b/nornir/plugins/connections/netmiko.py index 6ae3bed0..e5faff5d 100644 --- a/nornir/plugins/connections/netmiko.py +++ b/nornir/plugins/connections/netmiko.py @@ -30,7 +30,7 @@ def open( username: str, password: str, port: int, - device_type: str, + platform: str, connection_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: @@ -41,10 +41,10 @@ def open( "port": port, } - if device_type is not None: - # Look device_type up in corresponding map, if no entry return the host.nos unmodified - device_type = napalm_to_netmiko_map.get(device_type, device_type) - parameters["device_type"] = device_type + if platform is not None: + # Look platform up in corresponding map, if no entry return the host.nos unmodified + platform = napalm_to_netmiko_map.get(platform, platform) + parameters["device_type"] = platform netmiko_connection_args = connection_options or {} netmiko_connection_args.update(parameters) diff --git a/nornir/plugins/connections/paramiko.py b/nornir/plugins/connections/paramiko.py index abb4b521..ad17448f 100644 --- a/nornir/plugins/connections/paramiko.py +++ b/nornir/plugins/connections/paramiko.py @@ -23,7 +23,7 @@ def open( username: str, password: str, port: int, - device_type: str, + platform: str, connection_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: diff --git a/nornir/plugins/inventory/ansible.py b/nornir/plugins/inventory/ansible.py index 3d233dca..fa18b203 100644 --- a/nornir/plugins/inventory/ansible.py +++ b/nornir/plugins/inventory/ansible.py @@ -122,10 +122,10 @@ def read_vars_file(element: str, path: str, is_host: bool = True) -> VarsDict: @staticmethod def map_nornir_vars(obj: VarsDict): mappings = { - "ansible_host": "nornir_host", - "ansible_port": "nornir_ssh_port", - "ansible_user": "nornir_username", - "ansible_password": "nornir_password", + "ansible_host": "hostname", + "ansible_port": "port", + "ansible_user": "username", + "ansible_password": "password", } result = {} for k, v in obj.items(): diff --git a/nornir/plugins/inventory/netbox.py b/nornir/plugins/inventory/netbox.py index 133875de..d23d3dc1 100644 --- a/nornir/plugins/inventory/netbox.py +++ b/nornir/plugins/inventory/netbox.py @@ -35,7 +35,7 @@ def __init__( # Add value for IP address if d.get("primary_ip", {}): - temp["nornir_host"] = d["primary_ip"]["address"].split("/")[0] + temp["hostname"] = d["primary_ip"]["address"].split("/")[0] # Add values that don't have an option for 'slug' temp["serial"] = d["serial"] @@ -55,13 +55,13 @@ def __init__( temp["model"] = d["device_type"]["slug"] # Attempt to add 'platform' based of value in 'slug' - temp["nornir_nos"] = d["platform"]["slug"] if d["platform"] else None + temp["platform"] = d["platform"]["slug"] if d["platform"] else None else: temp["site"] = d["site"]["name"] temp["role"] = d["device_role"] temp["model"] = d["device_type"] - temp["nornir_nos"] = d["platform"] + temp["platform"] = d["platform"] # Assign temporary dict to outer dict devices[d["name"]] = temp diff --git a/tests/core/test_connections.py b/tests/core/test_connections.py index f7f59fbf..043618dd 100644 --- a/tests/core/test_connections.py +++ b/tests/core/test_connections.py @@ -22,7 +22,7 @@ def open( username: str, password: str, port: int, - device_type: str, + platform: str, connection_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: diff --git a/tests/core/test_filter.py b/tests/core/test_filter.py index adaa8fdf..33120d72 100644 --- a/tests/core/test_filter.py +++ b/tests/core/test_filter.py @@ -113,7 +113,7 @@ def test_filtering_by_attribute_name(self): assert filtered == ["dev1.group_1"] def test_filtering_string_in_list(self): - f = F(device_type__in=["linux", "mock"]) + f = F(platform__in=["linux", "mock"]) filtered = sorted(list((inventory.filter(f).hosts.keys()))) assert filtered == ["dev3.group_2", "dev4.group_2"] diff --git a/tests/core/test_inventory.py b/tests/core/test_inventory.py index c68aaa4a..bf2ea092 100644 --- a/tests/core/test_inventory.py +++ b/tests/core/test_inventory.py @@ -223,7 +223,7 @@ def test_to_dict(self): "www_server": "nginx", "role": "www", "port": 65001, - "device_type": "eos", + "platform": "eos", }, "dev3.group_2": { "name": "dev3.group_2", @@ -231,8 +231,8 @@ def test_to_dict(self): "www_server": "apache", "role": "www", "port": 65003, - "device_type": "linux", - "connection_options": {"napalm": {"device_type": "mock"}}, + "platform": "linux", + "connection_options": {"napalm": {"platform": "mock"}}, }, }, "groups": { diff --git a/tests/inventory_data/groups.yaml b/tests/inventory_data/groups.yaml index d094c207..7200d86d 100644 --- a/tests/inventory_data/groups.yaml +++ b/tests/inventory_data/groups.yaml @@ -4,7 +4,7 @@ defaults: hostname: 127.0.0.1 username: root password: docker - device_type: linux + platform: linux parent_group: a_var: blah diff --git a/tests/inventory_data/hosts.yaml b/tests/inventory_data/hosts.yaml index e3cd99db..772a3d06 100644 --- a/tests/inventory_data/hosts.yaml +++ b/tests/inventory_data/hosts.yaml @@ -6,7 +6,7 @@ dev1.group_1: www_server: nginx role: www port: 65001 - device_type: eos + platform: eos nested_data: a_dict: a: 1 @@ -19,7 +19,7 @@ dev2.group_1: - group_1 role: db port: 65002 - device_type: junos + platform: junos nested_data: a_dict: b: 2 @@ -33,10 +33,10 @@ dev3.group_2: www_server: apache role: www port: 65003 - device_type: linux + platform: linux connection_options: napalm: - device_type: mock + platform: mock dev4.group_2: groups: @@ -44,7 +44,7 @@ dev4.group_2: my_var: comes_from_dev4.group_2 role: db port: 65004 - device_type: linux + platform: linux connection_options: napalm: - device_type: mock + platform: mock diff --git a/tests/plugins/inventory/ansible/ini/expected/hosts.yaml b/tests/plugins/inventory/ansible/ini/expected/hosts.yaml index c8e69867..0fd8cdca 100644 --- a/tests/plugins/inventory/ansible/ini/expected/hosts.yaml +++ b/tests/plugins/inventory/ansible/ini/expected/hosts.yaml @@ -10,8 +10,8 @@ one.example.com: - dbservers my_var: from_one.example.com three.example.com: - nornir_host: 192.0.2.50 - nornir_ssh_port: 5555 + hostname: 192.0.2.50 + port: 5555 groups: - dbservers two.example.com: diff --git a/tests/plugins/inventory/ansible/yaml/expected/hosts.yaml b/tests/plugins/inventory/ansible/yaml/expected/hosts.yaml index c8e69867..0fd8cdca 100644 --- a/tests/plugins/inventory/ansible/yaml/expected/hosts.yaml +++ b/tests/plugins/inventory/ansible/yaml/expected/hosts.yaml @@ -10,8 +10,8 @@ one.example.com: - dbservers my_var: from_one.example.com three.example.com: - nornir_host: 192.0.2.50 - nornir_ssh_port: 5555 + hostname: 192.0.2.50 + port: 5555 groups: - dbservers two.example.com: diff --git a/tests/plugins/inventory/ansible/yaml2/expected/hosts.yaml b/tests/plugins/inventory/ansible/yaml2/expected/hosts.yaml index f516beff..9e9b70c0 100644 --- a/tests/plugins/inventory/ansible/yaml2/expected/hosts.yaml +++ b/tests/plugins/inventory/ansible/yaml2/expected/hosts.yaml @@ -1,8 +1,8 @@ one.example.com: groups: [] three.example.com: - nornir_host: 192.0.2.50 - nornir_ssh_port: 5555 + hostname: 192.0.2.50 + port: 5555 groups: [] two.example.com: groups: [] diff --git a/tests/plugins/inventory/netbox/2.3.5/expected.json b/tests/plugins/inventory/netbox/2.3.5/expected.json index f6c57ee1..c2312327 100644 --- a/tests/plugins/inventory/netbox/2.3.5/expected.json +++ b/tests/plugins/inventory/netbox/2.3.5/expected.json @@ -2,36 +2,36 @@ "hosts": { "1-Core": { "name": "1-Core", - "nornir_host": "10.0.1.1", + "hostname": "10.0.1.1", "serial": "", "vendor": "Juniper", "asset_tag": null, "site": "sunnyvale-ca", "role": "rt", "model": "mx480", - "nornir_nos": null + "platform": null }, "2-Distribution": { "name": "2-Distribution", - "nornir_host": "172.16.2.1", + "hostname": "172.16.2.1", "serial": "", "vendor": "Juniper", "asset_tag": null, "site": "sunnyvale-ca", "role": "rt", "model": "ex4550-32f", - "nornir_nos": null + "platform": null }, "3-Access": { "name": "3-Access", - "nornir_host": "192.168.3.1", + "hostname": "192.168.3.1", "serial": "", "vendor": "Cisco", "asset_tag": null, "site": "san-jose-ca", "role": "sw", "model": "3650-48tq-l", - "nornir_nos": null + "platform": null } }, "groups": { diff --git a/tests/plugins/inventory/netbox/2.3.5/expected_transform_function.json b/tests/plugins/inventory/netbox/2.3.5/expected_transform_function.json index ba70e1a2..21fe3639 100644 --- a/tests/plugins/inventory/netbox/2.3.5/expected_transform_function.json +++ b/tests/plugins/inventory/netbox/2.3.5/expected_transform_function.json @@ -2,36 +2,36 @@ "hosts": { "1-Core": { "name": "1-Core", - "nornir_host": "10.0.1.1", + "hostname": "10.0.1.1", "serial": "", "vendor": "Juniper", "asset_tag": null, "site": "sunnyvale-ca", "role": "rt", "model": "mx480", - "nornir_nos": "junos" + "platform": "junos" }, "2-Distribution": { "name": "2-Distribution", - "nornir_host": "172.16.2.1", + "hostname": "172.16.2.1", "serial": "", "vendor": "Juniper", "asset_tag": null, "site": "sunnyvale-ca", "role": "rt", "model": "ex4550-32f", - "nornir_nos": "junos" + "platform": "junos" }, "3-Access": { "name": "3-Access", - "nornir_host": "192.168.3.1", + "hostname": "192.168.3.1", "serial": "", "vendor": "Cisco", "asset_tag": null, "site": "san-jose-ca", "role": "sw", "model": "3650-48tq-l", - "nornir_nos": "ios" + "platform": "ios" } }, "groups": { diff --git a/tests/plugins/inventory/test_netbox.py b/tests/plugins/inventory/test_netbox.py index 8e632b43..9d7e82b6 100644 --- a/tests/plugins/inventory/test_netbox.py +++ b/tests/plugins/inventory/test_netbox.py @@ -22,7 +22,7 @@ def get_inv(requests_mock, case, **kwargs): def transform_function(host): vendor_map = {"Cisco": "ios", "Juniper": "junos"} - host["nornir_nos"] = vendor_map[host["vendor"]] + host["platform"] = vendor_map[host["vendor"]] class Test(object): From d1fc315445d9b53cc47baa69185a1327ffeba138 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Tue, 7 Aug 2018 10:57:00 +0200 Subject: [PATCH 020/109] black --- nornir/core/inventory.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 7731526d..cb0a1e99 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -316,9 +316,7 @@ def open_connection( username=username if username is not None else conn_params["username"], password=password if password is not None else conn_params["password"], port=port if port is not None else conn_params["port"], - platform=platform - if platform is not None - else conn_params["platform"], + platform=platform if platform is not None else conn_params["platform"], connection_options=connection_options if connection_options is not None else conn_params["connection_options"], From 57d6eb8eaada8143bbd0bfbfcdfa0830d8e6afd4 Mon Sep 17 00:00:00 2001 From: Erik Turk Date: Tue, 7 Aug 2018 10:06:23 -0400 Subject: [PATCH 021/109] Update inventory.ipynb typo --- docs/tutorials/intro/inventory.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/intro/inventory.ipynb b/docs/tutorials/intro/inventory.ipynb index 24611c5f..9507d5fe 100644 --- a/docs/tutorials/intro/inventory.ipynb +++ b/docs/tutorials/intro/inventory.ipynb @@ -264,7 +264,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The hosts file is basically a map where the outermost key is the hostname and then any arbitrary `` pair you want inside. Usually `nornir_*` keys have special meaning, you can investigate the [hosts](../../ref/api/inventory.rst#nornir.core.inventory.Host) class for details on those. In addition, the `groups` key is a list of groups you can inherite data from. We will inspect soon how the inheritance model works.\n", + "The hosts file is basically a map where the outermost key is the hostname and then any arbitrary `` pair you want inside. Usually `nornir_*` keys have special meaning, you can investigate the [hosts](../../ref/api/inventory.rst#nornir.core.inventory.Host) class for details on those. In addition, the `groups` key is a list of groups you can inherit data from. We will inspect soon how the inheritance model works.\n", "\n", "Now, let's look at the groups file:" ] From 5244e2dbd5118defdee0e0da60a9206bb69a1b4b Mon Sep 17 00:00:00 2001 From: Patrick Ogenstad Date: Wed, 8 Aug 2018 13:52:05 +0200 Subject: [PATCH 022/109] Remove old Python 2 import --- nornir/plugins/inventory/ansible.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nornir/plugins/inventory/ansible.py b/nornir/plugins/inventory/ansible.py index 3d233dca..a2e26101 100644 --- a/nornir/plugins/inventory/ansible.py +++ b/nornir/plugins/inventory/ansible.py @@ -1,8 +1,4 @@ -try: - import configparser as cp -except ImportError: - # https://github.com/python/mypy/issues/1153 - import ConfigParser as cp # type: ignore +import configparser as cp import logging import os from collections import defaultdict From d32b11e7e9ed64328c4f9ca6b97aa39779358a1b Mon Sep 17 00:00:00 2001 From: David Barroso Date: Thu, 9 Aug 2018 12:38:59 +0200 Subject: [PATCH 023/109] fix signature and rename configuration_options to advanced_options to avoid confusions --- nornir/core/connections.py | 12 +++++----- nornir/core/inventory.py | 14 ++++++------ nornir/plugins/connections/napalm.py | 22 +++++++++---------- nornir/plugins/connections/netmiko.py | 18 +++++++-------- nornir/plugins/connections/paramiko.py | 18 +++++++-------- tests/core/test_connections.py | 12 +++++----- .../tasks/networking/test_napalm_cli.py | 6 ++--- .../tasks/networking/test_napalm_configure.py | 12 +++++----- .../tasks/networking/test_napalm_get.py | 16 +++++++------- .../tasks/networking/test_napalm_validate.py | 10 ++++----- 10 files changed, 69 insertions(+), 71 deletions(-) diff --git a/nornir/core/connections.py b/nornir/core/connections.py index 1c03b39c..e40b95c0 100644 --- a/nornir/core/connections.py +++ b/nornir/core/connections.py @@ -25,12 +25,12 @@ def __init__(self) -> None: @abstractmethod def open( self, - hostname: str, - username: str, - password: str, - port: int, - platform: str, - connection_options: Optional[Dict[str, Any]] = None, + hostname: Optional[str], + username: Optional[str], + password: Optional[str], + port: Optional[int], + platform: Optional[str], + advanced_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: """ diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index cb0a1e99..81333eb8 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -230,7 +230,7 @@ def get_connection_parameters( "username": self.username, "password": self.password, "platform": self.platform, - "connection_options": {}, + "advanced_options": {}, } else: conn_params = self.get("connection_options", {}).get(connection, {}) @@ -240,7 +240,7 @@ def get_connection_parameters( "username": conn_params.get("username", self.username), "password": conn_params.get("password", self.password), "platform": conn_params.get("platform", self.platform), - "connection_options": conn_params.get("additional_options", {}), + "advanced_options": conn_params.get("advanced_options", {}), } def get_connection(self, connection: str) -> Any: @@ -289,7 +289,7 @@ def open_connection( password: Optional[str] = None, port: Optional[int] = None, platform: Optional[int] = None, - connection_options: Optional[int] = None, + advanced_options: Optional[int] = None, configuration: Optional[Config] = None, default_to_host_attributes: bool = True, ) -> None: @@ -317,9 +317,9 @@ def open_connection( password=password if password is not None else conn_params["password"], port=port if port is not None else conn_params["port"], platform=platform if platform is not None else conn_params["platform"], - connection_options=connection_options - if connection_options is not None - else conn_params["connection_options"], + advanced_options=advanced_options + if advanced_options is not None + else conn_params["advanced_options"], configuration=configuration if configuration is not None else self.nornir.config, @@ -331,7 +331,7 @@ def open_connection( password=password, port=port, platform=platform, - connection_options=connection_options, + advanced_options=advanced_options, configuration=configuration, ) return self.connections[connection] diff --git a/nornir/plugins/connections/napalm.py b/nornir/plugins/connections/napalm.py index e907b395..7a1d2aba 100644 --- a/nornir/plugins/connections/napalm.py +++ b/nornir/plugins/connections/napalm.py @@ -19,26 +19,24 @@ class Napalm(ConnectionPlugin): def open( self, - hostname: str, - username: str, - password: str, - port: int, - platform: str, - connection_options: Optional[Dict[str, Any]] = None, + hostname: Optional[str], + username: Optional[str], + password: Optional[str], + port: Optional[int], + platform: Optional[str], + advanced_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: - connection_options = connection_options or {} - if port: - connection_options["port"] = port + advanced_options = advanced_options or {} parameters = { "hostname": hostname, "username": username, "password": password, - "optional_args": connection_options or {}, + "optional_args": advanced_options or {}, } - if connection_options.get("timeout"): - parameters["timeout"] = connection_options["timeout"] + if advanced_options.get("timeout"): + parameters["timeout"] = advanced_options["timeout"] network_driver = get_network_driver(platform) connection = network_driver(**parameters) diff --git a/nornir/plugins/connections/netmiko.py b/nornir/plugins/connections/netmiko.py index e5faff5d..dc8bc5fd 100644 --- a/nornir/plugins/connections/netmiko.py +++ b/nornir/plugins/connections/netmiko.py @@ -26,12 +26,12 @@ class Netmiko(ConnectionPlugin): def open( self, - hostname: str, - username: str, - password: str, - port: int, - platform: str, - connection_options: Optional[Dict[str, Any]] = None, + hostname: Optional[str], + username: Optional[str], + password: Optional[str], + port: Optional[int], + platform: Optional[str], + advanced_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: parameters = { @@ -46,9 +46,9 @@ def open( platform = napalm_to_netmiko_map.get(platform, platform) parameters["device_type"] = platform - netmiko_connection_args = connection_options or {} - netmiko_connection_args.update(parameters) - self.connection = ConnectHandler(**netmiko_connection_args) + netmiko_advanced_args = advanced_options or {} + netmiko_advanced_args.update(parameters) + self.connection = ConnectHandler(**netmiko_advanced_args) def close(self) -> None: self.connection.disconnect() diff --git a/nornir/plugins/connections/paramiko.py b/nornir/plugins/connections/paramiko.py index ad17448f..94fd2a27 100644 --- a/nornir/plugins/connections/paramiko.py +++ b/nornir/plugins/connections/paramiko.py @@ -19,15 +19,15 @@ class Paramiko(ConnectionPlugin): def open( self, - hostname: str, - username: str, - password: str, - port: int, - platform: str, - connection_options: Optional[Dict[str, Any]] = None, + hostname: Optional[str], + username: Optional[str], + password: Optional[str], + port: Optional[int], + platform: Optional[str], + advanced_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: - connection_options = connection_options or {} + advanced_options = advanced_options or {} client = paramiko.SSHClient() client._policy = paramiko.WarningPolicy() @@ -61,8 +61,8 @@ def open( if "identityfile" in user_config: parameters["key_filename"] = user_config["identityfile"] - connection_options.update(parameters) - client.connect(**connection_options) + advanced_options.update(parameters) + client.connect(**advanced_options) self.connection = client def close(self) -> None: diff --git a/tests/core/test_connections.py b/tests/core/test_connections.py index 043618dd..3461d7f4 100644 --- a/tests/core/test_connections.py +++ b/tests/core/test_connections.py @@ -18,12 +18,12 @@ class DummyConnectionPlugin(ConnectionPlugin): def open( self, - hostname: str, - username: str, - password: str, - port: int, - platform: str, - connection_options: Optional[Dict[str, Any]] = None, + hostname: Optional[str], + username: Optional[str], + password: Optional[str], + port: Optional[int], + platform: Optional[str], + advanced_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: self.connection = True diff --git a/tests/plugins/tasks/networking/test_napalm_cli.py b/tests/plugins/tasks/networking/test_napalm_cli.py index 3236636c..d3479ae0 100644 --- a/tests/plugins/tasks/networking/test_napalm_cli.py +++ b/tests/plugins/tasks/networking/test_napalm_cli.py @@ -11,11 +11,11 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__)) + "/mocked/napalm_cli" -def connect(task, connection_options): +def connect(task, advanced_options): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", connection_options=connection_options, default_to_host_attributes=True + "napalm", advanced_options=advanced_options, default_to_host_attributes=True ) @@ -23,7 +23,7 @@ class Test(object): def test_napalm_cli(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_cli"} d = nornir.filter(name="dev3.group_2") - d.run(connect, connection_options=opt) + d.run(connect, advanced_options=opt) result = d.run( networking.napalm_cli, commands=["show version", "show interfaces"] ) diff --git a/tests/plugins/tasks/networking/test_napalm_configure.py b/tests/plugins/tasks/networking/test_napalm_configure.py index 45938cd3..d10002be 100644 --- a/tests/plugins/tasks/networking/test_napalm_configure.py +++ b/tests/plugins/tasks/networking/test_napalm_configure.py @@ -8,11 +8,11 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__)) + "/mocked/napalm_configure" -def connect(task, connection_options): +def connect(task, advanced_options): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", connection_options=connection_options, default_to_host_attributes=True + "napalm", advanced_options=advanced_options, default_to_host_attributes=True ) @@ -21,7 +21,7 @@ def test_napalm_configure_change_dry_run(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_configure_change_dry_run"} configuration = "hostname changed-hostname" d = nornir.filter(name="dev3.group_2") - d.run(connect, connection_options=opt) + d.run(connect, advanced_options=opt) result = d.run(networking.napalm_configure, configuration=configuration) assert result for h, r in result.items(): @@ -32,7 +32,7 @@ def test_napalm_configure_change_commit(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_configure_change_commit/step1"} configuration = "hostname changed-hostname" d = nornir.filter(name="dev3.group_2") - d.run(connect, connection_options=opt) + d.run(connect, advanced_options=opt) result = d.run( networking.napalm_configure, dry_run=False, configuration=configuration ) @@ -41,7 +41,7 @@ def test_napalm_configure_change_commit(self, nornir): assert "+hostname changed-hostname" in r.diff assert r.changed opt = {"path": THIS_DIR + "/test_napalm_configure_change_commit/step2"} - d.run(connect, connection_options=opt) + d.run(connect, advanced_options=opt) result = d.run( networking.napalm_configure, dry_run=True, configuration=configuration ) @@ -55,7 +55,7 @@ def test_napalm_configure_change_error(self, nornir): configuration = "hostname changed_hostname" d = nornir.filter(name="dev3.group_2") - d.run(connect, connection_options=opt) + d.run(connect, advanced_options=opt) results = d.run(networking.napalm_configure, configuration=configuration) processed = False for result in results.values(): diff --git a/tests/plugins/tasks/networking/test_napalm_get.py b/tests/plugins/tasks/networking/test_napalm_get.py index 0a7556a3..04241c64 100644 --- a/tests/plugins/tasks/networking/test_napalm_get.py +++ b/tests/plugins/tasks/networking/test_napalm_get.py @@ -6,11 +6,11 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__)) + "/mocked/napalm_get" -def connect(task, connection_options): +def connect(task, advanced_options): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", connection_options=connection_options, default_to_host_attributes=True + "napalm", advanced_options=advanced_options, default_to_host_attributes=True ) @@ -18,7 +18,7 @@ class Test(object): def test_napalm_getters(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters"} d = nornir.filter(name="dev3.group_2") - d.run(task=connect, connection_options=opt) + d.run(task=connect, advanced_options=opt) result = d.run(networking.napalm_get, getters=["facts", "interfaces"]) assert result for h, r in result.items(): @@ -28,7 +28,7 @@ def test_napalm_getters(self, nornir): def test_napalm_getters_error(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_error"} d = nornir.filter(name="dev3.group_2") - d.run(task=connect, connection_options=opt) + d.run(task=connect, advanced_options=opt) results = d.run(networking.napalm_get, getters=["facts", "interfaces"]) processed = False @@ -41,7 +41,7 @@ def test_napalm_getters_error(self, nornir): def test_napalm_getters_with_options_error(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_single_with_options"} d = nornir.filter(name="dev3.group_2") - d.run(task=connect, connection_options=opt) + d.run(task=connect, advanced_options=opt) result = d.run( task=networking.napalm_get, getters=["config"], nonexistent="asdsa" ) @@ -54,7 +54,7 @@ def test_napalm_getters_with_options_error(self, nornir): def test_napalm_getters_with_options_error_optional_args(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_single_with_options"} d = nornir.filter(name="dev3.group_2") - d.run(task=connect, connection_options=opt) + d.run(task=connect, advanced_options=opt) result = d.run( task=networking.napalm_get, getters=["config"], @@ -69,7 +69,7 @@ def test_napalm_getters_with_options_error_optional_args(self, nornir): def test_napalm_getters_single_with_options(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_single_with_options"} d = nornir.filter(name="dev3.group_2") - d.run(task=connect, connection_options=opt) + d.run(task=connect, advanced_options=opt) result = d.run( task=networking.napalm_get, getters=["config"], retrieve="candidate" ) @@ -81,7 +81,7 @@ def test_napalm_getters_single_with_options(self, nornir): def test_napalm_getters_multiple_with_options(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_multiple_with_options"} d = nornir.filter(name="dev3.group_2") - d.run(task=connect, connection_options=opt) + d.run(task=connect, advanced_options=opt) result = d.run( task=networking.napalm_get, getters=["config", "facts"], diff --git a/tests/plugins/tasks/networking/test_napalm_validate.py b/tests/plugins/tasks/networking/test_napalm_validate.py index 6ee79742..559a92ee 100644 --- a/tests/plugins/tasks/networking/test_napalm_validate.py +++ b/tests/plugins/tasks/networking/test_napalm_validate.py @@ -6,11 +6,11 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__)) -def connect(task, connection_options): +def connect(task, advanced_options): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", connection_options=connection_options, default_to_host_attributes=True + "napalm", advanced_options=advanced_options, default_to_host_attributes=True ) @@ -18,7 +18,7 @@ class Test(object): def test_napalm_validate_src_ok(self, nornir): opt = {"path": THIS_DIR + "/mocked/napalm_get/test_napalm_getters"} d = nornir.filter(name="dev3.group_2") - d.run(connect, connection_options=opt) + d.run(connect, advanced_options=opt) result = d.run( networking.napalm_validate, src=THIS_DIR + "/data/validate_ok.yaml" ) @@ -29,7 +29,7 @@ def test_napalm_validate_src_ok(self, nornir): def test_napalm_validate_src_error(self, nornir): opt = {"path": THIS_DIR + "/mocked/napalm_get/test_napalm_getters"} d = nornir.filter(name="dev3.group_2") - d.run(connect, connection_options=opt) + d.run(connect, advanced_options=opt) result = d.run( networking.napalm_validate, src=THIS_DIR + "/data/validate_error.yaml" @@ -42,7 +42,7 @@ def test_napalm_validate_src_error(self, nornir): def test_napalm_validate_src_validate_source(self, nornir): opt = {"path": THIS_DIR + "/mocked/napalm_get/test_napalm_getters"} d = nornir.filter(name="dev3.group_2") - d.run(connect, connection_options=opt) + d.run(connect, advanced_options=opt) validation_dict = [{"get_interfaces": {"Ethernet1": {"description": ""}}}] From 4825e23df04decc5bb590a2a07e1960ed6c2696d Mon Sep 17 00:00:00 2001 From: David Barroso Date: Thu, 9 Aug 2018 13:07:49 +0200 Subject: [PATCH 024/109] add tests to validate parameters resolution --- tests/core/test_connections.py | 48 +++++++++++++++++++++++++++++++++ tests/inventory_data/hosts.yaml | 6 +++++ 2 files changed, 54 insertions(+) diff --git a/tests/core/test_connections.py b/tests/core/test_connections.py index 3461d7f4..a6b1dac7 100644 --- a/tests/core/test_connections.py +++ b/tests/core/test_connections.py @@ -28,6 +28,13 @@ def open( ) -> None: self.connection = True self.state["something"] = "something" + self.hostname = hostname + self.username = username + self.password = password + self.port = port + self.platform = platform + self.advanced_options = advanced_options + self.configuration = configuration def close(self) -> None: self.connection = False @@ -64,6 +71,12 @@ def a_task(task): task.host.get_connection("dummy") +def validate_params(task, conn, params): + task.host.get_connection(conn) + for k, v in params.items(): + assert getattr(task.host.connections[conn], k) == v + + class Test(object): def test_open_and_close_connection(self, nornir): nornir.data.available_connections["dummy"] = DummyConnectionPlugin @@ -93,3 +106,38 @@ def test_context_manager(self, nornir): assert "dummy" in nr.inventory.hosts["dev2.group_1"].connections assert "dummy" not in nr.inventory.hosts["dev2.group_1"].connections nornir.data.reset_failed_hosts() + + def test_validate_params_simple(self, nornir): + nornir.data.available_connections["dummy_no_overrides"] = DummyConnectionPlugin + params = { + "hostname": "127.0.0.1", + "username": "root", + "password": "docker", + "port": 65002, + "platform": "junos", + "advanced_options": {}, + } + nr = nornir.filter(name="dev2.group_1") + r = nr.run( + task=validate_params, + conn="dummy_no_overrides", + params=params, + num_workers=1, + ) + assert len(r) == 1 + assert not r.failed + + def test_validate_params_overrides(self, nornir): + nornir.data.available_connections["dummy"] = DummyConnectionPlugin + params = { + "hostname": "overriden_hostname", + "username": "root", + "password": "docker", + "port": None, + "platform": "junos", + "advanced_options": {"awesome_feature": 1}, + } + nr = nornir.filter(name="dev2.group_1") + r = nr.run(task=validate_params, conn="dummy", params=params, num_workers=1) + assert len(r) == 1 + assert not r.failed diff --git a/tests/inventory_data/hosts.yaml b/tests/inventory_data/hosts.yaml index 772a3d06..209c9397 100644 --- a/tests/inventory_data/hosts.yaml +++ b/tests/inventory_data/hosts.yaml @@ -26,6 +26,12 @@ dev2.group_1: c: 3 a_list: [2, 3] a_string: "qwe" + connection_options: + dummy: + hostname: overriden_hostname + port: null + advanced_options: + awesome_feature: 1 dev3.group_2: groups: From 46d7b3d6e7332e9a22fec1ca8e0ee351c16d6bc8 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Thu, 9 Aug 2018 13:58:25 +0200 Subject: [PATCH 025/109] fix breaking change from ruamel.yaml --- nornir/core/inventory.py | 5 +++-- tests/plugins/tasks/data/test_load_yaml.py | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 81333eb8..93cd5393 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -5,6 +5,7 @@ from nornir.core.connections import Connections from nornir.core.exceptions import ConnectionAlreadyOpen, ConnectionNotOpen +from ruamel.yaml.comments import CommentedMap VarsDict = Dict[str, Any] HostsDict = Dict[str, VarsDict] @@ -389,7 +390,7 @@ def __init__( for group_name, group_details in groups.items(): if group_details is None: group = Group(name=group_name, nornir=nornir) - elif isinstance(group_details, dict): + elif isinstance(group_details, (dict, CommentedMap)): group = Group(name=group_name, nornir=nornir, **group_details) elif isinstance(group_details, Group): group = group_details @@ -407,7 +408,7 @@ def __init__( self.hosts = {} for n, h in hosts.items(): - if isinstance(h, dict): + if isinstance(h, (dict, CommentedMap)): h = Host(name=n, nornir=nornir, defaults=self.defaults, **h) if transform_function: diff --git a/tests/plugins/tasks/data/test_load_yaml.py b/tests/plugins/tasks/data/test_load_yaml.py index 971964a0..f60a1215 100644 --- a/tests/plugins/tasks/data/test_load_yaml.py +++ b/tests/plugins/tasks/data/test_load_yaml.py @@ -17,7 +17,6 @@ def test_load_yaml(self, nornir): d = r.result assert d["env"] == "test" assert d["services"] == ["dhcp", "dns"] - assert isinstance(d["a_dict"], dict) def test_load_yaml_error_broken_file(self, nornir): test_file = "{}/broken.yaml".format(data_dir) From b437401a5525cfe0a85361df8b6f5527d21bd0c0 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Fri, 10 Aug 2018 14:27:33 +0200 Subject: [PATCH 026/109] fixes to documentation --- docs/tutorials/intro/executing_tasks.ipynb | 60 +++++------ docs/tutorials/intro/failed_tasks.ipynb | 12 +-- docs/tutorials/intro/grouping_tasks.ipynb | 106 +++++++----------- docs/tutorials/intro/inventory.ipynb | 112 ++++++++++---------- docs/tutorials/intro/inventory/Vagrantfile | 6 +- docs/tutorials/intro/inventory/hosts.yaml | 104 +++++++++--------- docs/tutorials/intro/task_results.ipynb | 42 ++++---- nornir/core/configuration.py | 4 +- nornir/core/inventory.py | 7 +- nornir/plugins/inventory/simple.py | 40 +++---- nornir/plugins/tasks/networking/tcp_ping.py | 2 +- tests/core/test_connections.py | 10 -- tests/inventory_data/external_hosts.yaml | 4 +- tests/plugins/inventory/test_nsot.py | 6 +- 14 files changed, 237 insertions(+), 278 deletions(-) diff --git a/docs/tutorials/intro/executing_tasks.ipynb b/docs/tutorials/intro/executing_tasks.ipynb index fb9dd0ec..b6e50133 100644 --- a/docs/tutorials/intro/executing_tasks.ipynb +++ b/docs/tutorials/intro/executing_tasks.ipynb @@ -34,19 +34,19 @@ "text": [ "\u001b[1m\u001b[36mremote_command******************************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[34m* host1.cmh ** changed : False *************************************************\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[32m---- remote_command ** changed : False ----------------------------------------\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32mvvvv remote_command ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0mtotal 8\n", - "drwxrwxrwt 2 root root 4096 Mar 25 17:26 .\n", - "drwxr-xr-x 24 root root 4096 Mar 25 17:26 ..\n", - "\u001b[0m\n", + "drwxrwxrwt 2 root root 4096 Aug 10 11:47 .\n", + "drwxr-xr-x 24 root root 4096 Aug 10 11:47 ..\n", "\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m^^^^ END remote_command ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[34m* host2.cmh ** changed : False *************************************************\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[32m---- remote_command ** changed : False ----------------------------------------\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32mvvvv remote_command ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0mtotal 8\n", - "drwxrwxrwt 2 root root 4096 Mar 25 17:27 .\n", - "drwxr-xr-x 24 root root 4096 Mar 25 17:27 ..\n", - "\u001b[0m\n", + "drwxrwxrwt 2 root root 4096 Aug 10 11:48 .\n", + "drwxr-xr-x 24 root root 4096 Aug 10 11:48 ..\n", "\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m^^^^ END remote_command ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", "\u001b[0m" ] } @@ -86,18 +86,18 @@ "text": [ "\u001b[1m\u001b[36mnapalm_get**********************************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[34m* spine00.bma ** changed : False ***********************************************\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[32m---- napalm_get ** changed : False --------------------------------------------\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32mvvvv napalm_get ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m{\u001b[0m \u001b[0m'facts'\u001b[0m: \u001b[0m{\u001b[0m \u001b[0m'fqdn'\u001b[0m: \u001b[0m'localhost'\u001b[0m,\n", " \u001b[0m'hostname'\u001b[0m: \u001b[0m'localhost'\u001b[0m,\n", " \u001b[0m'interface_list'\u001b[0m: \u001b[0m['Ethernet1', 'Ethernet2', 'Management1']\u001b[0m,\n", " \u001b[0m'model'\u001b[0m: \u001b[0m'vEOS'\u001b[0m,\n", - " \u001b[0m'os_version'\u001b[0m: \u001b[0m'4.17.5M-4414219.4175M'\u001b[0m,\n", + " \u001b[0m'os_version'\u001b[0m: \u001b[0m'4.20.1F-6820520.4201F'\u001b[0m,\n", " \u001b[0m'serial_number'\u001b[0m: \u001b[0m''\u001b[0m,\n", - " \u001b[0m'uptime'\u001b[0m: \u001b[0m441\u001b[0m,\n", + " \u001b[0m'uptime'\u001b[0m: \u001b[0m77764\u001b[0m,\n", " \u001b[0m'vendor'\u001b[0m: \u001b[0m'Arista'\u001b[0m}\u001b[0m}\u001b[0m\n", - "\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m^^^^ END napalm_get ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[34m* spine01.bma ** changed : False ***********************************************\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[32m---- napalm_get ** changed : False --------------------------------------------\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32mvvvv napalm_get ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m{\u001b[0m \u001b[0m'facts'\u001b[0m: \u001b[0m{\u001b[0m \u001b[0m'fqdn'\u001b[0m: \u001b[0m'vsrx'\u001b[0m,\n", " \u001b[0m'hostname'\u001b[0m: \u001b[0m'vsrx'\u001b[0m,\n", " \u001b[0m'interface_list'\u001b[0m: \u001b[0m[\u001b[0m \u001b[0m\u001b[0m'ge-0/0/0'\u001b[0m,\n", @@ -127,10 +127,10 @@ " \u001b[0m'vlan'\u001b[0m]\u001b[0m,\n", " \u001b[0m'model'\u001b[0m: \u001b[0m'FIREFLY-PERIMETER'\u001b[0m,\n", " \u001b[0m'os_version'\u001b[0m: \u001b[0m'12.1X47-D20.7'\u001b[0m,\n", - " \u001b[0m'serial_number'\u001b[0m: \u001b[0m'b4321e51218e'\u001b[0m,\n", - " \u001b[0m'uptime'\u001b[0m: \u001b[0m334\u001b[0m,\n", + " \u001b[0m'serial_number'\u001b[0m: \u001b[0m'7287f12c493d'\u001b[0m,\n", + " \u001b[0m'uptime'\u001b[0m: \u001b[0m77648\u001b[0m,\n", " \u001b[0m'vendor'\u001b[0m: \u001b[0m'Juniper'\u001b[0m}\u001b[0m}\u001b[0m\n", - "\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m^^^^ END napalm_get ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", "\u001b[0m" ] } @@ -218,8 +218,8 @@ "text": [ "\u001b[1m\u001b[36mavailable_resources*************************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[34m* host1.cmh ** changed : False *************************************************\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[32m---- available_resources ** changed : False -----------------------------------\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[32m---- Available disk ** changed : False ----------------------------------------\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32mvvvv available_resources ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m---- Available disk ** changed : False ----------------------------------------- INFO\u001b[0m\n", "\u001b[0mFilesystem Size Used Avail Use% Mounted on\n", "/dev/mapper/precise64-root 79G 2.2G 73G 3% /\n", "udev 174M 4.0K 174M 1% /dev\n", @@ -227,18 +227,18 @@ "none 5.0M 0 5.0M 0% /run/lock\n", "none 183M 0 183M 0% /run/shm\n", "/dev/sda1 228M 25M 192M 12% /boot\n", - "vagrant 373G 215G 159G 58% /vagrant\n", + "vagrant 373G 271G 103G 73% /vagrant\n", "\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[32m---- Available memory ** changed : False --------------------------------------\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m---- Available memory ** changed : False --------------------------------------- INFO\u001b[0m\n", "\u001b[0m total used free shared buffers cached\n", - "Mem: 365 88 277 0 8 36\n", - "-/+ buffers/cache: 43 322\n", + "Mem: 365 87 278 0 8 36\n", + "-/+ buffers/cache: 42 323\n", "Swap: 767 0 767\n", "\u001b[0m\n", - "\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m^^^^ END available_resources ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[34m* host2.cmh ** changed : False *************************************************\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[32m---- available_resources ** changed : False -----------------------------------\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[32m---- Available disk ** changed : False ----------------------------------------\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32mvvvv available_resources ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m---- Available disk ** changed : False ----------------------------------------- INFO\u001b[0m\n", "\u001b[0mFilesystem Size Used Avail Use% Mounted on\n", "/dev/mapper/precise64-root 79G 2.2G 73G 3% /\n", "udev 174M 4.0K 174M 1% /dev\n", @@ -246,15 +246,15 @@ "none 5.0M 0 5.0M 0% /run/lock\n", "none 183M 0 183M 0% /run/shm\n", "/dev/sda1 228M 25M 192M 12% /boot\n", - "vagrant 373G 215G 159G 58% /vagrant\n", + "vagrant 373G 271G 103G 73% /vagrant\n", "\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[32m---- Available memory ** changed : False --------------------------------------\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m---- Available memory ** changed : False --------------------------------------- INFO\u001b[0m\n", "\u001b[0m total used free shared buffers cached\n", - "Mem: 365 93 271 0 9 36\n", - "-/+ buffers/cache: 48 317\n", + "Mem: 365 88 277 0 8 36\n", + "-/+ buffers/cache: 42 322\n", "Swap: 767 0 767\n", "\u001b[0m\n", - "\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m^^^^ END available_resources ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", "\u001b[0m" ] } diff --git a/docs/tutorials/intro/failed_tasks.ipynb b/docs/tutorials/intro/failed_tasks.ipynb index ed7d31c8..26a60b14 100644 --- a/docs/tutorials/intro/failed_tasks.ipynb +++ b/docs/tutorials/intro/failed_tasks.ipynb @@ -106,8 +106,8 @@ { "data": { "text/plain": [ - "{'leaf00.cmh': MultiResult: [Result: \"basic_configuration\", Result: \"Base Configuration\", Result: \"Loading Configuration on the device\"],\n", - " 'spine00.cmh': MultiResult: [Result: \"basic_configuration\", Result: \"Base Configuration\", Result: \"Loading Configuration on the device\"]}" + "{'spine00.cmh': MultiResult: [Result: \"basic_configuration\", Result: \"Base Configuration\", Result: \"Loading Configuration on the device\"],\n", + " 'leaf00.cmh': MultiResult: [Result: \"basic_configuration\", Result: \"Base Configuration\", Result: \"Loading Configuration on the device\"]}" ] }, "execution_count": 5, @@ -171,9 +171,9 @@ "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", - " File \"/Users/dbarroso/workspace/nornir/nornir/core/task.py\", line 62, in start\n", + " File \"/Users/dbarroso/workspace/nornir/nornir/core/task.py\", line 63, in start\n", " r = self.task(self, **self.params)\n", - " File \"/Users/dbarroso/workspace/nornir/nornir/plugins/tasks/networking/napalm_configure.py\", line 26, in napalm_configure\n", + " File \"/Users/dbarroso/workspace/nornir/nornir/plugins/tasks/networking/napalm_configure.py\", line 32, in napalm_configure\n", " device.load_merge_candidate(filename=filename, config=configuration)\n", " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.6/site-packages/napalm/eos/eos.py\", line 245, in load_merge_candidate\n", " self._load_config(filename, config, False)\n", @@ -219,9 +219,9 @@ "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", - " File \"/Users/dbarroso/workspace/nornir/nornir/core/task.py\", line 62, in start\n", + " File \"/Users/dbarroso/workspace/nornir/nornir/core/task.py\", line 63, in start\n", " r = self.task(self, **self.params)\n", - " File \"/Users/dbarroso/workspace/nornir/nornir/plugins/tasks/networking/napalm_configure.py\", line 26, in napalm_configure\n", + " File \"/Users/dbarroso/workspace/nornir/nornir/plugins/tasks/networking/napalm_configure.py\", line 32, in napalm_configure\n", " device.load_merge_candidate(filename=filename, config=configuration)\n", " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.6/site-packages/napalm/eos/eos.py\", line 245, in load_merge_candidate\n", " self._load_config(filename, config, False)\n", diff --git a/docs/tutorials/intro/grouping_tasks.ipynb b/docs/tutorials/intro/grouping_tasks.ipynb index c2afe0a2..973e49b7 100644 --- a/docs/tutorials/intro/grouping_tasks.ipynb +++ b/docs/tutorials/intro/grouping_tasks.ipynb @@ -49,7 +49,7 @@ " r = task.run(task=text.template_file,\n", " name=\"Base Configuration\",\n", " template=\"base.j2\",\n", - " path=f\"templates/{task.host.nos}\")\n", + " path=f\"templates/{task.host.platform}\")\n", " \n", " # Save the compiled configuration into a host variable\n", " task.host[\"config\"] = r.result\n", @@ -131,10 +131,10 @@ "text": [ "\u001b[1m\u001b[32m**** Playbook to configure the network *****************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[36mbasic_configuration*************************************************************\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* spine00.cmh ** changed : True ************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* leaf00.cmh ** changed : True *************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m---- Base Configuration ** changed : False ------------------------------------- INFO\u001b[0m\n", - "\u001b[0mhostname spine00.cmh\n", + "\u001b[0mhostname leaf00.cmh\n", "ip domain-name cmh.acme.local\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", "\u001b[0m@@ -7,6 +7,9 @@\n", @@ -142,29 +142,29 @@ " !\n", " transceiver qsfp default-mode 4x10G\n", "+!\n", - "+hostname spine00.cmh\n", + "+hostname leaf00.cmh\n", "+ip domain-name cmh.acme.local\n", " !\n", " spanning-tree mode mstp\n", " !\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* spine01.cmh ** changed : True ************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* leaf01.cmh ** changed : True *************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m---- Base Configuration ** changed : False ------------------------------------- INFO\u001b[0m\n", "\u001b[0msystem {\n", - " host-name spine01.cmh;\n", + " host-name leaf01.cmh;\n", " domain-name cmh.acme.local;\n", "}\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", "\u001b[0m[edit system]\n", "- host-name vsrx;\n", - "+ host-name spine01.cmh;\n", + "+ host-name leaf01.cmh;\n", "+ domain-name cmh.acme.local;\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* leaf00.cmh ** changed : True *************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* spine00.cmh ** changed : True ************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m---- Base Configuration ** changed : False ------------------------------------- INFO\u001b[0m\n", - "\u001b[0mhostname leaf00.cmh\n", + "\u001b[0mhostname spine00.cmh\n", "ip domain-name cmh.acme.local\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", "\u001b[0m@@ -7,6 +7,9 @@\n", @@ -172,23 +172,23 @@ " !\n", " transceiver qsfp default-mode 4x10G\n", "+!\n", - "+hostname leaf00.cmh\n", + "+hostname spine00.cmh\n", "+ip domain-name cmh.acme.local\n", " !\n", " spanning-tree mode mstp\n", " !\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* leaf01.cmh ** changed : True *************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* spine01.cmh ** changed : True ************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m---- Base Configuration ** changed : False ------------------------------------- INFO\u001b[0m\n", "\u001b[0msystem {\n", - " host-name leaf01.cmh;\n", + " host-name spine01.cmh;\n", " domain-name cmh.acme.local;\n", "}\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", "\u001b[0m[edit system]\n", "- host-name vsrx;\n", - "+ host-name leaf01.cmh;\n", + "+ host-name spine01.cmh;\n", "+ domain-name cmh.acme.local;\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", "\u001b[0m" @@ -210,7 +210,7 @@ "However, this was a `dry_run`. Let's set the `dry_run` variable to `False` so changes are commited and then run the code again:\n", "\n", "
\n", - "**Note:** The `dry_run` value is shared between the main nornir objects and its childs so in the snippet below `nr.dry_run = False` and `cmh.dry_run = False` are equivalent.\n", + "**Note:** The `dry_run` value is shared between the main nornir objects and its childs so in the snippet below `nr.data.dry_run = False` and `cmh.data.dry_run = False` are equivalent.\n", "
" ] }, @@ -225,10 +225,10 @@ "text": [ "\u001b[1m\u001b[32m**** Playbook to configure the network *****************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[36mbasic_configuration*************************************************************\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* spine00.cmh ** changed : True ************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* leaf00.cmh ** changed : True *************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m---- Base Configuration ** changed : False ------------------------------------- INFO\u001b[0m\n", - "\u001b[0mhostname spine00.cmh\n", + "\u001b[0mhostname leaf00.cmh\n", "ip domain-name cmh.acme.local\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", "\u001b[0m@@ -7,6 +7,9 @@\n", @@ -236,29 +236,29 @@ " !\n", " transceiver qsfp default-mode 4x10G\n", "+!\n", - "+hostname spine00.cmh\n", + "+hostname leaf00.cmh\n", "+ip domain-name cmh.acme.local\n", " !\n", " spanning-tree mode mstp\n", " !\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* spine01.cmh ** changed : True ************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* leaf01.cmh ** changed : True *************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m---- Base Configuration ** changed : False ------------------------------------- INFO\u001b[0m\n", "\u001b[0msystem {\n", - " host-name spine01.cmh;\n", + " host-name leaf01.cmh;\n", " domain-name cmh.acme.local;\n", "}\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", "\u001b[0m[edit system]\n", "- host-name vsrx;\n", - "+ host-name spine01.cmh;\n", + "+ host-name leaf01.cmh;\n", "+ domain-name cmh.acme.local;\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* leaf00.cmh ** changed : True *************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* spine00.cmh ** changed : True ************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m---- Base Configuration ** changed : False ------------------------------------- INFO\u001b[0m\n", - "\u001b[0mhostname leaf00.cmh\n", + "\u001b[0mhostname spine00.cmh\n", "ip domain-name cmh.acme.local\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", "\u001b[0m@@ -7,6 +7,9 @@\n", @@ -266,23 +266,23 @@ " !\n", " transceiver qsfp default-mode 4x10G\n", "+!\n", - "+hostname leaf00.cmh\n", + "+hostname spine00.cmh\n", "+ip domain-name cmh.acme.local\n", " !\n", " spanning-tree mode mstp\n", " !\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* leaf01.cmh ** changed : True *************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* spine01.cmh ** changed : True ************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m---- Base Configuration ** changed : False ------------------------------------- INFO\u001b[0m\n", "\u001b[0msystem {\n", - " host-name leaf01.cmh;\n", + " host-name spine01.cmh;\n", " domain-name cmh.acme.local;\n", "}\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", "\u001b[0m[edit system]\n", "- host-name vsrx;\n", - "+ host-name leaf01.cmh;\n", + "+ host-name spine01.cmh;\n", "+ domain-name cmh.acme.local;\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", "\u001b[0m" @@ -290,7 +290,7 @@ } ], "source": [ - "nr.dry_run = False\n", + "nr.data.dry_run = False\n", "print_title(\"Playbook to configure the network\")\n", "result = cmh.run(task=basic_configuration)\n", "print_result(result)" @@ -314,65 +314,37 @@ "text": [ "\u001b[1m\u001b[32m**** Playbook to configure the network *****************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[36mbasic_configuration*************************************************************\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* spine00.cmh ** changed : True ************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* leaf00.cmh ** changed : False ************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m---- Base Configuration ** changed : False ------------------------------------- INFO\u001b[0m\n", - "\u001b[0mhostname spine00.cmh\n", + "\u001b[0mhostname leaf00.cmh\n", "ip domain-name cmh.acme.local\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", - "\u001b[0m@@ -7,6 +7,9 @@\n", - " action bash sudo /mnt/flash/initialize_ma1.sh\n", - " !\n", - " transceiver qsfp default-mode 4x10G\n", - "+!\n", - "+hostname spine00.cmh\n", - "+ip domain-name cmh.acme.local\n", - " !\n", - " spanning-tree mode mstp\n", - " !\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m---- Loading Configuration on the device ** changed : False -------------------- INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* spine01.cmh ** changed : True ************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* leaf01.cmh ** changed : False ************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m---- Base Configuration ** changed : False ------------------------------------- INFO\u001b[0m\n", "\u001b[0msystem {\n", - " host-name spine01.cmh;\n", + " host-name leaf01.cmh;\n", " domain-name cmh.acme.local;\n", "}\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", - "\u001b[0m[edit system]\n", - "- host-name vsrx;\n", - "+ host-name spine01.cmh;\n", - "+ domain-name cmh.acme.local;\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m---- Loading Configuration on the device ** changed : False -------------------- INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* leaf00.cmh ** changed : True *************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* spine00.cmh ** changed : False ***********************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m---- Base Configuration ** changed : False ------------------------------------- INFO\u001b[0m\n", - "\u001b[0mhostname leaf00.cmh\n", + "\u001b[0mhostname spine00.cmh\n", "ip domain-name cmh.acme.local\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", - "\u001b[0m@@ -7,6 +7,9 @@\n", - " action bash sudo /mnt/flash/initialize_ma1.sh\n", - " !\n", - " transceiver qsfp default-mode 4x10G\n", - "+!\n", - "+hostname leaf00.cmh\n", - "+ip domain-name cmh.acme.local\n", - " !\n", - " spanning-tree mode mstp\n", - " !\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m---- Loading Configuration on the device ** changed : False -------------------- INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* leaf01.cmh ** changed : True *************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* spine01.cmh ** changed : False ***********************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m---- Base Configuration ** changed : False ------------------------------------- INFO\u001b[0m\n", "\u001b[0msystem {\n", - " host-name leaf01.cmh;\n", + " host-name spine01.cmh;\n", " domain-name cmh.acme.local;\n", "}\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", - "\u001b[0m[edit system]\n", - "- host-name vsrx;\n", - "+ host-name leaf01.cmh;\n", - "+ domain-name cmh.acme.local;\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m---- Loading Configuration on the device ** changed : False -------------------- INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", "\u001b[0m" ] diff --git a/docs/tutorials/intro/inventory.ipynb b/docs/tutorials/intro/inventory.ipynb index 8bbadabb..cc4fa550 100644 --- a/docs/tutorials/intro/inventory.ipynb +++ b/docs/tutorials/intro/inventory.ipynb @@ -107,15 +107,15 @@ "\n", "
  1 ---\n",
        "  2 host1.cmh:\n",
-       "  3     nornir_host: 127.0.0.1\n",
-       "  4     nornir_ssh_port: 2201\n",
-       "  5     nornir_username: vagrant\n",
-       "  6     nornir_password: vagrant\n",
+       "  3     hostname: 127.0.0.1\n",
+       "  4     port: 2201\n",
+       "  5     username: vagrant\n",
+       "  6     password: vagrant\n",
        "  7     site: cmh\n",
        "  8     role: host\n",
        "  9     groups:\n",
        " 10         - cmh\n",
-       " 11     nornir_nos: linux\n",
+       " 11     platform: linux\n",
        " 12     type: host\n",
        " 13     nested_data:\n",
        " 14         a_dict:\n",
@@ -125,15 +125,15 @@
        " 18         a_string: "asdasd"\n",
        " 19 \n",
        " 20 host2.cmh:\n",
-       " 21     nornir_host: 127.0.0.1\n",
-       " 22     nornir_ssh_port: 2202\n",
-       " 23     nornir_username: vagrant\n",
-       " 24     nornir_password: vagrant\n",
+       " 21     hostname: 127.0.0.1\n",
+       " 22     port: 2202\n",
+       " 23     username: vagrant\n",
+       " 24     password: vagrant\n",
        " 25     site: cmh\n",
        " 26     role: host\n",
        " 27     groups:\n",
        " 28         - cmh\n",
-       " 29     nornir_nos: linux\n",
+       " 29     platform: linux\n",
        " 30     type: host\n",
        " 31     nested_data:\n",
        " 32         a_dict:\n",
@@ -143,52 +143,52 @@
        " 36         a_string: "qwe"\n",
        " 37 \n",
        " 38 spine00.cmh:\n",
-       " 39     nornir_host: 127.0.0.1\n",
-       " 40     nornir_username: vagrant\n",
-       " 41     nornir_password: vagrant\n",
-       " 42     nornir_network_api_port: 12444\n",
+       " 39     hostname: 127.0.0.1\n",
+       " 40     username: vagrant\n",
+       " 41     password: vagrant\n",
+       " 42     port: 12444\n",
        " 43     site: cmh\n",
        " 44     role: spine\n",
        " 45     groups:\n",
        " 46         - cmh\n",
-       " 47     nornir_nos: eos\n",
+       " 47     platform: eos\n",
        " 48     type: network_device\n",
        " 49 \n",
        " 50 spine01.cmh:\n",
-       " 51     nornir_host: 127.0.0.1\n",
-       " 52     nornir_username: vagrant\n",
-       " 53     nornir_password: ""\n",
-       " 54     nornir_network_api_port: 12204\n",
+       " 51     hostname: 127.0.0.1\n",
+       " 52     username: vagrant\n",
+       " 53     password: ""\n",
+       " 54     port: 12204\n",
        " 55     site: cmh\n",
        " 56     role: spine\n",
        " 57     groups:\n",
        " 58         - cmh\n",
-       " 59     nornir_nos: junos\n",
+       " 59     platform: junos\n",
        " 60     type: network_device\n",
        " 61 \n",
        " 62 leaf00.cmh:\n",
-       " 63     nornir_host: 127.0.0.1\n",
-       " 64     nornir_username: vagrant\n",
-       " 65     nornir_password: vagrant\n",
-       " 66     nornir_network_api_port: 12443\n",
+       " 63     hostname: 127.0.0.1\n",
+       " 64     username: vagrant\n",
+       " 65     password: vagrant\n",
+       " 66     port: 12443\n",
        " 67     site: cmh\n",
        " 68     role: leaf\n",
        " 69     groups:\n",
        " 70         - cmh\n",
-       " 71     nornir_nos: eos\n",
+       " 71     platform: eos\n",
        " 72     type: network_device\n",
        " 73     asn: 65100\n",
        " 74 \n",
        " 75 leaf01.cmh:\n",
-       " 76     nornir_host: 127.0.0.1\n",
-       " 77     nornir_username: vagrant\n",
-       " 78     nornir_password: ""\n",
-       " 79     nornir_network_api_port: 12203\n",
+       " 76     hostname: 127.0.0.1\n",
+       " 77     username: vagrant\n",
+       " 78     password: ""\n",
+       " 79     port: 12203\n",
        " 80     site: cmh\n",
        " 81     role: leaf\n",
        " 82     groups:\n",
        " 83         - cmh\n",
-       " 84     nornir_nos: junos\n",
+       " 84     platform: junos\n",
        " 85     type: network_device\n",
        " 86     asn: 65101\n",
        " 87 \n",
@@ -197,7 +197,7 @@
        " 90     role: host\n",
        " 91     groups:\n",
        " 92         - bma\n",
-       " 93     nornir_nos: linux\n",
+       " 93     platform: linux\n",
        " 94     type: host\n",
        " 95 \n",
        " 96 host2.bma:\n",
@@ -205,55 +205,55 @@
        " 98     role: host\n",
        " 99     groups:\n",
        "100         - bma\n",
-       "101     nornir_nos: linux\n",
+       "101     platform: linux\n",
        "102     type: host\n",
        "103 \n",
        "104 spine00.bma:\n",
-       "105     nornir_host: 127.0.0.1\n",
-       "106     nornir_username: vagrant\n",
-       "107     nornir_password: vagrant\n",
-       "108     nornir_network_api_port: 12444\n",
+       "105     hostname: 127.0.0.1\n",
+       "106     username: vagrant\n",
+       "107     password: vagrant\n",
+       "108     port: 12444\n",
        "109     site: bma\n",
        "110     role: spine\n",
        "111     groups:\n",
        "112         - bma\n",
-       "113     nornir_nos: eos\n",
+       "113     platform: eos\n",
        "114     type: network_device\n",
        "115 \n",
        "116 spine01.bma:\n",
-       "117     nornir_host: 127.0.0.1\n",
-       "118     nornir_username: vagrant\n",
-       "119     nornir_password: ""\n",
-       "120     nornir_network_api_port: 12204\n",
+       "117     hostname: 127.0.0.1\n",
+       "118     username: vagrant\n",
+       "119     password: ""\n",
+       "120     port: 12204\n",
        "121     site: bma\n",
        "122     role: spine\n",
        "123     groups:\n",
        "124         - bma\n",
-       "125     nornir_nos: junos\n",
+       "125     platform: junos\n",
        "126     type: network_device\n",
        "127 \n",
        "128 leaf00.bma:\n",
-       "129     nornir_host: 127.0.0.1\n",
-       "130     nornir_username: vagrant\n",
-       "131     nornir_password: vagrant\n",
-       "132     nornir_network_api_port: 12443\n",
+       "129     hostname: 127.0.0.1\n",
+       "130     username: vagrant\n",
+       "131     password: vagrant\n",
+       "132     port: 12443\n",
        "133     site: bma\n",
        "134     role: leaf\n",
        "135     groups:\n",
        "136         - bma\n",
-       "137     nornir_nos: eos\n",
+       "137     platform: eos\n",
        "138     type: network_device\n",
        "139 \n",
        "140 leaf01.bma:\n",
-       "141     nornir_host: 127.0.0.1\n",
-       "142     nornir_username: vagrant\n",
-       "143     nornir_password: wrong_password\n",
-       "144     nornir_network_api_port: 12203\n",
+       "141     hostname: 127.0.0.1\n",
+       "142     username: vagrant\n",
+       "143     password: wrong_password\n",
+       "144     port: 12203\n",
        "145     site: bma\n",
        "146     role: leaf\n",
        "147     groups:\n",
        "148         - bma\n",
-       "149     nornir_nos: junos\n",
+       "149     platform: junos\n",
        "150     type: network_device\n",
        "
\n", "\n" @@ -276,7 +276,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The hosts file is basically a map where the outermost key is the hostname and then any arbitrary `` pair you want inside. Usually `nornir_*` keys have special meaning, you can investigate the [hosts](../../ref/api/inventory.rst#nornir.core.inventory.Host) class for details on those. In addition, the `groups` key is a list of groups you can inherite data from. We will inspect soon how the inheritance model works.\n", + "The hosts file is basically a map where the outermost key is the hostname and then any arbitrary `` pair you want inside. Some keys like hostname, username or password have special meaning, you can investigate the [hosts](../../ref/api/inventory.rst#nornir.core.inventory.Host) class for details on those. In addition, the `groups` key is a list of groups you can inherite data from. We will inspect soon how the inheritance model works.\n", "\n", "Now, let's look at the groups file:" ] @@ -420,7 +420,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 4, @@ -531,7 +531,7 @@ { "data": { "text/plain": [ - "dict_keys(['name', 'groups', 'nornir_host', 'nornir_username', 'nornir_password', 'nornir_network_api_port', 'site', 'role', 'nornir_nos', 'type', 'asn', 'domain'])" + "dict_keys(['name', 'groups', 'hostname', 'username', 'password', 'port', 'site', 'role', 'platform', 'type', 'asn', 'domain'])" ] }, "execution_count": 8, @@ -1099,7 +1099,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "dict_keys(['host1.cmh', 'host2.cmh', 'spine00.cmh', 'leaf00.cmh', 'host1.bma', 'host2.bma', 'spine00.bma', 'leaf00.bma'])\n" + "dict_keys([])\n" ] } ], diff --git a/docs/tutorials/intro/inventory/Vagrantfile b/docs/tutorials/intro/inventory/Vagrantfile index b0c2f21a..a38af3ea 100644 --- a/docs/tutorials/intro/inventory/Vagrantfile +++ b/docs/tutorials/intro/inventory/Vagrantfile @@ -2,7 +2,7 @@ # vi: set ft=ruby : """ You will need the boxes: - * vEOS-4.17.5M + * vEOS-lab-4.20.1F * JunOS - juniper/ffp-12.1X47-D20.7-packetmode * To provision and test JunOS first you have to add the ssh vagrant ssh key into the ssh-agent. I.e.: ssh-add /opt/vagrant/embedded/gems/gems/vagrant-`vagrant --version | awk '{ print $2 }'`/keys/vagrant @@ -12,7 +12,7 @@ Vagrant.configure(2) do |config| config.vbguest.auto_update = false config.vm.define "spine00" do |spine00| - spine00.vm.box = "vEOS-lab-4.17.5M" + spine00.vm.box = "vEOS-lab-4.20.1F" spine00.vm.network :forwarded_port, guest: 443, host: 12444, id: 'https' @@ -31,7 +31,7 @@ Vagrant.configure(2) do |config| config.vm.define "leaf00" do |leaf00| - leaf00.vm.box = "vEOS-lab-4.17.5M" + leaf00.vm.box = "vEOS-lab-4.20.1F" leaf00.vm.network :forwarded_port, guest: 443, host: 12443, id: 'https' diff --git a/docs/tutorials/intro/inventory/hosts.yaml b/docs/tutorials/intro/inventory/hosts.yaml index d48d5c21..748b0d70 100644 --- a/docs/tutorials/intro/inventory/hosts.yaml +++ b/docs/tutorials/intro/inventory/hosts.yaml @@ -1,14 +1,14 @@ --- host1.cmh: - nornir_host: 127.0.0.1 - nornir_ssh_port: 2201 - nornir_username: vagrant - nornir_password: vagrant + hostname: 127.0.0.1 + port: 2201 + username: vagrant + password: vagrant site: cmh role: host groups: - cmh - nornir_nos: linux + platform: linux type: host nested_data: a_dict: @@ -18,15 +18,15 @@ host1.cmh: a_string: "asdasd" host2.cmh: - nornir_host: 127.0.0.1 - nornir_ssh_port: 2202 - nornir_username: vagrant - nornir_password: vagrant + hostname: 127.0.0.1 + port: 2202 + username: vagrant + password: vagrant site: cmh role: host groups: - cmh - nornir_nos: linux + platform: linux type: host nested_data: a_dict: @@ -36,52 +36,52 @@ host2.cmh: a_string: "qwe" spine00.cmh: - nornir_host: 127.0.0.1 - nornir_username: vagrant - nornir_password: vagrant - nornir_network_api_port: 12444 + hostname: 127.0.0.1 + username: vagrant + password: vagrant + port: 12444 site: cmh role: spine groups: - cmh - nornir_nos: eos + platform: eos type: network_device spine01.cmh: - nornir_host: 127.0.0.1 - nornir_username: vagrant - nornir_password: "" - nornir_network_api_port: 12204 + hostname: 127.0.0.1 + username: vagrant + password: "" + port: 12204 site: cmh role: spine groups: - cmh - nornir_nos: junos + platform: junos type: network_device leaf00.cmh: - nornir_host: 127.0.0.1 - nornir_username: vagrant - nornir_password: vagrant - nornir_network_api_port: 12443 + hostname: 127.0.0.1 + username: vagrant + password: vagrant + port: 12443 site: cmh role: leaf groups: - cmh - nornir_nos: eos + platform: eos type: network_device asn: 65100 leaf01.cmh: - nornir_host: 127.0.0.1 - nornir_username: vagrant - nornir_password: "" - nornir_network_api_port: 12203 + hostname: 127.0.0.1 + username: vagrant + password: "" + port: 12203 site: cmh role: leaf groups: - cmh - nornir_nos: junos + platform: junos type: network_device asn: 65101 @@ -90,7 +90,7 @@ host1.bma: role: host groups: - bma - nornir_nos: linux + platform: linux type: host host2.bma: @@ -98,53 +98,53 @@ host2.bma: role: host groups: - bma - nornir_nos: linux + platform: linux type: host spine00.bma: - nornir_host: 127.0.0.1 - nornir_username: vagrant - nornir_password: vagrant - nornir_network_api_port: 12444 + hostname: 127.0.0.1 + username: vagrant + password: vagrant + port: 12444 site: bma role: spine groups: - bma - nornir_nos: eos + platform: eos type: network_device spine01.bma: - nornir_host: 127.0.0.1 - nornir_username: vagrant - nornir_password: "" - nornir_network_api_port: 12204 + hostname: 127.0.0.1 + username: vagrant + password: "" + port: 12204 site: bma role: spine groups: - bma - nornir_nos: junos + platform: junos type: network_device leaf00.bma: - nornir_host: 127.0.0.1 - nornir_username: vagrant - nornir_password: vagrant - nornir_network_api_port: 12443 + hostname: 127.0.0.1 + username: vagrant + password: vagrant + port: 12443 site: bma role: leaf groups: - bma - nornir_nos: eos + platform: eos type: network_device leaf01.bma: - nornir_host: 127.0.0.1 - nornir_username: vagrant - nornir_password: wrong_password - nornir_network_api_port: 12203 + hostname: 127.0.0.1 + username: vagrant + password: wrong_password + port: 12203 site: bma role: leaf groups: - bma - nornir_nos: junos + platform: junos type: network_device diff --git a/docs/tutorials/intro/task_results.ipynb b/docs/tutorials/intro/task_results.ipynb index 978c6a11..048a20c6 100644 --- a/docs/tutorials/intro/task_results.ipynb +++ b/docs/tutorials/intro/task_results.ipynb @@ -30,7 +30,7 @@ " r = task.run(task=text.template_file,\n", " name=\"Base Configuration\",\n", " template=\"base.j2\",\n", - " path=f\"templates/{task.host.nos}\",\n", + " path=f\"templates/{task.host.platform}\",\n", " severity_level=logging.DEBUG)\n", "\n", " # Save the compiled configuration into a host variable\n", @@ -79,7 +79,7 @@ "output_type": "stream", "text": [ "\u001b[1m\u001b[36mbasic_configuration*************************************************************\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* spine00.cmh ** changed : True ************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* leaf00.cmh ** changed : True *************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", "\u001b[0m@@ -7,6 +7,9 @@\n", @@ -87,21 +87,21 @@ " !\n", " transceiver qsfp default-mode 4x10G\n", "+!\n", - "+hostname spine00.cmh\n", + "+hostname leaf00.cmh\n", "+ip domain-name cmh.acme.local\n", " !\n", " spanning-tree mode mstp\n", " !\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* spine01.cmh ** changed : True ************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* leaf01.cmh ** changed : True *************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", "\u001b[0m[edit system]\n", "- host-name vsrx;\n", - "+ host-name spine01.cmh;\n", + "+ host-name leaf01.cmh;\n", "+ domain-name cmh.acme.local;\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* leaf00.cmh ** changed : True *************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* spine00.cmh ** changed : True ************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", "\u001b[0m@@ -7,6 +7,9 @@\n", @@ -109,18 +109,18 @@ " !\n", " transceiver qsfp default-mode 4x10G\n", "+!\n", - "+hostname leaf00.cmh\n", + "+hostname spine00.cmh\n", "+ip domain-name cmh.acme.local\n", " !\n", " spanning-tree mode mstp\n", " !\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* leaf01.cmh ** changed : True *************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* spine01.cmh ** changed : True ************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", "\u001b[0m[edit system]\n", "- host-name vsrx;\n", - "+ host-name leaf01.cmh;\n", + "+ host-name spine01.cmh;\n", "+ domain-name cmh.acme.local;\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", "\u001b[0m" @@ -224,10 +224,10 @@ "output_type": "stream", "text": [ "\u001b[1m\u001b[36mbasic_configuration*************************************************************\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* spine00.cmh ** changed : True ************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* leaf00.cmh ** changed : True *************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m---- Base Configuration ** changed : False ------------------------------------- DEBUG\u001b[0m\n", - "\u001b[0mhostname spine00.cmh\n", + "\u001b[0mhostname leaf00.cmh\n", "ip domain-name cmh.acme.local\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", "\u001b[0m@@ -7,6 +7,9 @@\n", @@ -235,29 +235,29 @@ " !\n", " transceiver qsfp default-mode 4x10G\n", "+!\n", - "+hostname spine00.cmh\n", + "+hostname leaf00.cmh\n", "+ip domain-name cmh.acme.local\n", " !\n", " spanning-tree mode mstp\n", " !\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* spine01.cmh ** changed : True ************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* leaf01.cmh ** changed : True *************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m---- Base Configuration ** changed : False ------------------------------------- DEBUG\u001b[0m\n", "\u001b[0msystem {\n", - " host-name spine01.cmh;\n", + " host-name leaf01.cmh;\n", " domain-name cmh.acme.local;\n", "}\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", "\u001b[0m[edit system]\n", "- host-name vsrx;\n", - "+ host-name spine01.cmh;\n", + "+ host-name leaf01.cmh;\n", "+ domain-name cmh.acme.local;\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* leaf00.cmh ** changed : True *************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* spine00.cmh ** changed : True ************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m---- Base Configuration ** changed : False ------------------------------------- DEBUG\u001b[0m\n", - "\u001b[0mhostname leaf00.cmh\n", + "\u001b[0mhostname spine00.cmh\n", "ip domain-name cmh.acme.local\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", "\u001b[0m@@ -7,6 +7,9 @@\n", @@ -265,23 +265,23 @@ " !\n", " transceiver qsfp default-mode 4x10G\n", "+!\n", - "+hostname leaf00.cmh\n", + "+hostname spine00.cmh\n", "+ip domain-name cmh.acme.local\n", " !\n", " spanning-tree mode mstp\n", " !\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[34m* leaf01.cmh ** changed : True *************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* spine01.cmh ** changed : True ************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv basic_configuration ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m---- Base Configuration ** changed : False ------------------------------------- DEBUG\u001b[0m\n", "\u001b[0msystem {\n", - " host-name leaf01.cmh;\n", + " host-name spine01.cmh;\n", " domain-name cmh.acme.local;\n", "}\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[33m---- Loading Configuration on the device ** changed : True --------------------- INFO\u001b[0m\n", "\u001b[0m[edit system]\n", "- host-name vsrx;\n", - "+ host-name leaf01.cmh;\n", + "+ host-name spine01.cmh;\n", "+ domain-name cmh.acme.local;\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", "\u001b[0m" diff --git a/nornir/core/configuration.py b/nornir/core/configuration.py index 49973d32..79bc2e87 100644 --- a/nornir/core/configuration.py +++ b/nornir/core/configuration.py @@ -13,9 +13,7 @@ }, "transform_function": { "description": "Path to transform function. The transform_function you provide " - "will run against each host in the inventory. This is useful to manipulate host " - "data and make it more consumable. For instance, if your inventory has a 'user' " - "attribute you could use this function to map it to 'nornir_user'", + "will run against each host in the inventory.", "type": "str", "default": {}, }, diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 93cd5393..3b8be594 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -208,12 +208,12 @@ def port(self): @property def username(self): - """Either ``nornir_username`` or user running the script.""" + """Either ``username`` or user running the script.""" return self.get("username", getpass.getuser()) @property def password(self): - """Either ``nornir_password`` or empty string.""" + """Either ``password`` or empty string.""" return self.get("password", "") @property @@ -370,8 +370,7 @@ class Inventory(object): groups (dict): keys are group names and values are either :obj:`Group` or a dict representing the group data. transform_function (callable): we will call this function for each host. This is useful - to manipulate host data and make it more consumable. For instance, if your inventory - has a "user" attribute you could use this function to map it to "nornir_user" + to manipulate host data and make it more consumable. Attributes: hosts (dict): keys are hostnames and values are :obj:`Host`. diff --git a/nornir/plugins/inventory/simple.py b/nornir/plugins/inventory/simple.py index 1946788d..bee4ea93 100644 --- a/nornir/plugins/inventory/simple.py +++ b/nornir/plugins/inventory/simple.py @@ -20,72 +20,72 @@ class SimpleInventory(Inventory): role: host groups: - cmh-host - nos: linux + platform: linux host2.cmh: site: cmh role: host groups: - cmh-host - nos: linux + platform: linux switch00.cmh: - nornir_ip: 127.0.0.1 - nornir_username: vagrant - nornir_password: vagrant + hostname: 127.0.0.1 + username: vagrant + password: vagrant napalm_port: 12443 site: cmh role: leaf groups: - cmh-leaf - nos: eos + platform: eos switch01.cmh: - nornir_ip: 127.0.0.1 - nornir_username: vagrant - nornir_password: "" + hostname: 127.0.0.1 + username: vagrant + password: "" napalm_port: 12203 site: cmh role: leaf groups: - cmh-leaf - nos: junos + platform: juplatform host1.bma: site: bma role: host groups: - bma-host - nos: linux + platform: linux host2.bma: site: bma role: host groups: - bma-host - nos: linux + platform: linux switch00.bma: - nornir_ip: 127.0.0.1 - nornir_username: vagrant - nornir_password: vagrant + hostname: 127.0.0.1 + username: vagrant + password: vagrant napalm_port: 12443 site: bma role: leaf groups: - bma-leaf - nos: eos + platform: eos switch01.bma: - nornir_ip: 127.0.0.1 - nornir_username: vagrant - nornir_password: "" + hostname: 127.0.0.1 + username: vagrant + password: "" napalm_port: 12203 site: bma role: leaf groups: - bma-leaf - nos: junos + platform: juplatform * group file:: diff --git a/nornir/plugins/tasks/networking/tcp_ping.py b/nornir/plugins/tasks/networking/tcp_ping.py index be254845..ab19122c 100644 --- a/nornir/plugins/tasks/networking/tcp_ping.py +++ b/nornir/plugins/tasks/networking/tcp_ping.py @@ -14,7 +14,7 @@ def tcp_ping( Arguments: ports (list of int): tcp ports to ping timeout (int, optional): defaults to 2 - host (string, optional): defaults to ``nornir_ip`` + host (string, optional): defaults to ``hostname`` Returns: diff --git a/tests/core/test_connections.py b/tests/core/test_connections.py index a6b1dac7..14341a9d 100644 --- a/tests/core/test_connections.py +++ b/tests/core/test_connections.py @@ -6,16 +6,6 @@ class DummyConnectionPlugin(ConnectionPlugin): - """ - This plugin connects to the device using the NAPALM driver and sets the - relevant connection. - - Inventory: - napalm_options: maps directly to ``optional_args`` when establishing the connection - nornir_network_api_port: maps to ``optional_args["port"]`` - napalm_options["timeout"]: maps to ``timeout``. - """ - def open( self, hostname: Optional[str], diff --git a/tests/inventory_data/external_hosts.yaml b/tests/inventory_data/external_hosts.yaml index 03b6f6dc..139d98b6 100644 --- a/tests/inventory_data/external_hosts.yaml +++ b/tests/inventory_data/external_hosts.yaml @@ -1,6 +1,6 @@ --- www.github.com: - nornir_ip: www.github.com + hostname: www.github.com dummy: - nornir_ip: 1.1.1.1 + hostname: 1.1.1.1 diff --git a/tests/plugins/inventory/test_nsot.py b/tests/plugins/inventory/test_nsot.py index c5c5d9da..5c91261c 100644 --- a/tests/plugins/inventory/test_nsot.py +++ b/tests/plugins/inventory/test_nsot.py @@ -25,7 +25,7 @@ def transform_function(host): attrs = ["user", "password"] for a in attrs: if a in host.data: - host["nornir_{}".format(a)] = host.data[a] + host["modified_{}".format(a)] = host.data[a] class Test(object): @@ -39,5 +39,5 @@ def test_inventory(self, requests_mock): def test_transform_function(self, requests_mock): inv = get_inv(requests_mock, "1.3.0", transform_function=transform_function) for host in inv.hosts.values(): - assert host["user"] == host["nornir_user"] - assert host["password"] == host["nornir_password"] + assert host["user"] == host["modified_user"] + assert host["password"] == host["modified_password"] From 0a4d26ecce58b2b38da1ab6da6387bf37f7a5ee1 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 12 Aug 2018 13:13:28 +0200 Subject: [PATCH 027/109] pass _options without requiring indentation --- nornir/core/inventory.py | 2 +- nornir/plugins/connections/napalm.py | 14 ++++++++----- nornir/plugins/connections/netmiko.py | 9 ++++---- nornir/plugins/connections/paramiko.py | 3 +-- tests/core/test_inventory.py | 2 +- tests/inventory_data/hosts.yaml | 21 ++++++++----------- .../tasks/networking/test_napalm_cli.py | 4 +++- .../tasks/networking/test_napalm_configure.py | 4 +++- .../tasks/networking/test_napalm_get.py | 4 +++- .../tasks/networking/test_napalm_validate.py | 4 +++- .../plugins/tasks/networking/test_tcp_ping.py | 2 +- 11 files changed, 38 insertions(+), 31 deletions(-) diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 3b8be594..1fff9f88 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -234,7 +234,7 @@ def get_connection_parameters( "advanced_options": {}, } else: - conn_params = self.get("connection_options", {}).get(connection, {}) + conn_params = self.get(f"{connection}_options", {}) return { "hostname": conn_params.get("hostname", self.hostname), "port": conn_params.get("port", self.port), diff --git a/nornir/plugins/connections/napalm.py b/nornir/plugins/connections/napalm.py index 7a1d2aba..a1e86ea6 100644 --- a/nornir/plugins/connections/napalm.py +++ b/nornir/plugins/connections/napalm.py @@ -12,9 +12,8 @@ class Napalm(ConnectionPlugin): relevant connection. Inventory: - napalm_options: maps directly to ``optional_args`` when establishing the connection - nornir_network_api_port: maps to ``optional_args["port"]`` - napalm_options["timeout"]: maps to ``timeout``. + advanced_options: passed as it is to the napalm driver + advanced_options["timeout"]: maps to ``timeout``. """ def open( @@ -29,12 +28,17 @@ def open( ) -> None: advanced_options = advanced_options or {} - parameters = { + parameters: Dict[str, Any] = { "hostname": hostname, "username": username, "password": password, - "optional_args": advanced_options or {}, + "optional_args": {}, } + parameters.update(advanced_options) + + if port and "port" not in advanced_options: + parameters["optional_args"]["port"] = port + if advanced_options.get("timeout"): parameters["timeout"] = advanced_options["timeout"] diff --git a/nornir/plugins/connections/netmiko.py b/nornir/plugins/connections/netmiko.py index dc8bc5fd..cc16963f 100644 --- a/nornir/plugins/connections/netmiko.py +++ b/nornir/plugins/connections/netmiko.py @@ -20,8 +20,7 @@ class Netmiko(ConnectionPlugin): relevant connection. Inventory: - netmiko_options: maps to argument passed to ``ConnectHandler``. - nornir_network_ssh_port: maps to ``port`` + advanced_options: maps to argument passed to ``ConnectHandler``. """ def open( @@ -46,9 +45,9 @@ def open( platform = napalm_to_netmiko_map.get(platform, platform) parameters["device_type"] = platform - netmiko_advanced_args = advanced_options or {} - netmiko_advanced_args.update(parameters) - self.connection = ConnectHandler(**netmiko_advanced_args) + advanced_options = advanced_options or {} + parameters.update(advanced_options) + self.connection = ConnectHandler(**parameters) def close(self) -> None: self.connection.disconnect() diff --git a/nornir/plugins/connections/paramiko.py b/nornir/plugins/connections/paramiko.py index 94fd2a27..a03779f6 100644 --- a/nornir/plugins/connections/paramiko.py +++ b/nornir/plugins/connections/paramiko.py @@ -13,8 +13,7 @@ class Paramiko(ConnectionPlugin): relevant connection. Inventory: - paramiko_options: maps to argument passed to ``ConnectHandler``. - nornir_network_ssh_port: maps to ``port`` + advanced_options: maps to argument passed to ``ConnectHandler``. """ def open( diff --git a/tests/core/test_inventory.py b/tests/core/test_inventory.py index bf2ea092..ebb9ef2a 100644 --- a/tests/core/test_inventory.py +++ b/tests/core/test_inventory.py @@ -232,7 +232,7 @@ def test_to_dict(self): "role": "www", "port": 65003, "platform": "linux", - "connection_options": {"napalm": {"platform": "mock"}}, + "napalm_options": {"platform": "mock"}, }, }, "groups": { diff --git a/tests/inventory_data/hosts.yaml b/tests/inventory_data/hosts.yaml index 209c9397..c2d6f46d 100644 --- a/tests/inventory_data/hosts.yaml +++ b/tests/inventory_data/hosts.yaml @@ -26,12 +26,11 @@ dev2.group_1: c: 3 a_list: [2, 3] a_string: "qwe" - connection_options: - dummy: - hostname: overriden_hostname - port: null - advanced_options: - awesome_feature: 1 + dummy_options: + hostname: overriden_hostname + port: null + advanced_options: + awesome_feature: 1 dev3.group_2: groups: @@ -40,9 +39,8 @@ dev3.group_2: role: www port: 65003 platform: linux - connection_options: - napalm: - platform: mock + napalm_options: + platform: mock dev4.group_2: groups: @@ -51,6 +49,5 @@ dev4.group_2: role: db port: 65004 platform: linux - connection_options: - napalm: - platform: mock + napalm_options: + platform: mock diff --git a/tests/plugins/tasks/networking/test_napalm_cli.py b/tests/plugins/tasks/networking/test_napalm_cli.py index d3479ae0..d508e680 100644 --- a/tests/plugins/tasks/networking/test_napalm_cli.py +++ b/tests/plugins/tasks/networking/test_napalm_cli.py @@ -15,7 +15,9 @@ def connect(task, advanced_options): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", advanced_options=advanced_options, default_to_host_attributes=True + "napalm", + advanced_options={"optional_args": advanced_options}, + default_to_host_attributes=True, ) diff --git a/tests/plugins/tasks/networking/test_napalm_configure.py b/tests/plugins/tasks/networking/test_napalm_configure.py index d10002be..1260ab82 100644 --- a/tests/plugins/tasks/networking/test_napalm_configure.py +++ b/tests/plugins/tasks/networking/test_napalm_configure.py @@ -12,7 +12,9 @@ def connect(task, advanced_options): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", advanced_options=advanced_options, default_to_host_attributes=True + "napalm", + advanced_options={"optional_args": advanced_options}, + default_to_host_attributes=True, ) diff --git a/tests/plugins/tasks/networking/test_napalm_get.py b/tests/plugins/tasks/networking/test_napalm_get.py index 04241c64..191ec14a 100644 --- a/tests/plugins/tasks/networking/test_napalm_get.py +++ b/tests/plugins/tasks/networking/test_napalm_get.py @@ -10,7 +10,9 @@ def connect(task, advanced_options): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", advanced_options=advanced_options, default_to_host_attributes=True + "napalm", + advanced_options={"optional_args": advanced_options}, + default_to_host_attributes=True, ) diff --git a/tests/plugins/tasks/networking/test_napalm_validate.py b/tests/plugins/tasks/networking/test_napalm_validate.py index 559a92ee..e30cb285 100644 --- a/tests/plugins/tasks/networking/test_napalm_validate.py +++ b/tests/plugins/tasks/networking/test_napalm_validate.py @@ -10,7 +10,9 @@ def connect(task, advanced_options): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", advanced_options=advanced_options, default_to_host_attributes=True + "napalm", + advanced_options={"optional_args": advanced_options}, + default_to_host_attributes=True, ) diff --git a/tests/plugins/tasks/networking/test_tcp_ping.py b/tests/plugins/tasks/networking/test_tcp_ping.py index af509a5d..dc9f42ea 100644 --- a/tests/plugins/tasks/networking/test_tcp_ping.py +++ b/tests/plugins/tasks/networking/test_tcp_ping.py @@ -58,4 +58,4 @@ def test_tcp_ping_external_hosts(): assert r.result[443] else: assert r.result[23] is False - assert r.result[443] is False + assert r.result[443] From 32a85313ec733c68a8a150c8cab0e259de34341f Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 12 Aug 2018 14:21:05 +0200 Subject: [PATCH 028/109] added tutorial on how to handle connections --- docs/howto/config.yaml | 6 + docs/howto/handling_connections.ipynb | 174 ++++++++++++++++++++ docs/howto/handling_connections/get_facts.1 | 16 ++ docs/howto/inventory/groups.yaml | 2 + docs/howto/inventory/hosts.yaml | 7 + 5 files changed, 205 insertions(+) create mode 100644 docs/howto/config.yaml create mode 100644 docs/howto/handling_connections.ipynb create mode 100644 docs/howto/handling_connections/get_facts.1 create mode 100644 docs/howto/inventory/groups.yaml create mode 100644 docs/howto/inventory/hosts.yaml diff --git a/docs/howto/config.yaml b/docs/howto/config.yaml new file mode 100644 index 00000000..ae8ee1ea --- /dev/null +++ b/docs/howto/config.yaml @@ -0,0 +1,6 @@ +--- +inventory: nornir.plugins.inventory.simple.SimpleInventory +SimpleInventory: + host_file: "inventory/hosts.yaml" + group_file: "inventory/groups.yaml" + diff --git a/docs/howto/handling_connections.ipynb b/docs/howto/handling_connections.ipynb new file mode 100644 index 00000000..005c2f3b --- /dev/null +++ b/docs/howto/handling_connections.ipynb @@ -0,0 +1,174 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# How to handle connections to devices\n", + "\n", + "## Automatically\n", + "\n", + "By default, connections are handled automatically:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from nornir.core import InitNornir\n", + "from nornir.plugins.functions.text import print_result\n", + "from nornir.plugins.tasks.networking import napalm_get" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1m\u001b[36mnapalm_get**********************************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* rtr00 ** changed : False *****************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32mvvvv napalm_get ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", + "\u001b[0m{\u001b[0m \u001b[0m'facts'\u001b[0m: \u001b[0m{\u001b[0m \u001b[0m'fqdn'\u001b[0m: \u001b[0m'localhost'\u001b[0m,\n", + " \u001b[0m'hostname'\u001b[0m: \u001b[0m'localhost'\u001b[0m,\n", + " \u001b[0m'interface_list'\u001b[0m: \u001b[0m[\u001b[0m \u001b[0m\u001b[0m'Ethernet1'\u001b[0m,\n", + " \u001b[0m'Ethernet2'\u001b[0m,\n", + " \u001b[0m'Ethernet3'\u001b[0m,\n", + " \u001b[0m'Ethernet4'\u001b[0m,\n", + " \u001b[0m'Management1'\u001b[0m]\u001b[0m,\n", + " \u001b[0m'model'\u001b[0m: \u001b[0m'vEOS'\u001b[0m,\n", + " \u001b[0m'os_version'\u001b[0m: \u001b[0m'4.15.5M-3054042.4155M'\u001b[0m,\n", + " \u001b[0m'serial_number'\u001b[0m: \u001b[0m''\u001b[0m,\n", + " \u001b[0m'uptime'\u001b[0m: \u001b[0m'...'\u001b[0m,\n", + " \u001b[0m'vendor'\u001b[0m: \u001b[0m'Arista'\u001b[0m}\u001b[0m}\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m^^^^ END napalm_get ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", + "\u001b[0m" + ] + } + ], + "source": [ + "nr = InitNornir(config_file=\"config.yaml\")\n", + "rtr = nr.filter(name=\"rtr00\")\n", + "r = rtr.run(\n", + " task=napalm_get,\n", + " getters=[\"facts\"]\n", + ")\n", + "print_result(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Manually\n", + "\n", + "In some circumstances, you may want to manage connections manually. To do so you can use \n", + "[open_connection](../ref/api/inventory.rst#nornir.core.inventory.Host.open_connection), [close_connection](../ref/api/inventory.rst#nornir.core.inventory.Host.close_connection), [close_connections](../ref/api/inventory.rst#nornir.core.inventory.Host.close_connections) and [Nornir.close_connections](../ref/api/nornir.rst#nornir.core.Nornir.close_connections). For instance:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1m\u001b[36mtask_manages_connection_manually************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* rtr00 ** changed : False *****************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32mvvvv task_manages_connection_manually ** changed : False vvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m---- napalm_get ** changed : False --------------------------------------------- INFO\u001b[0m\n", + "\u001b[0m{\u001b[0m \u001b[0m'facts'\u001b[0m: \u001b[0m{\u001b[0m \u001b[0m'fqdn'\u001b[0m: \u001b[0m'localhost'\u001b[0m,\n", + " \u001b[0m'hostname'\u001b[0m: \u001b[0m'localhost'\u001b[0m,\n", + " \u001b[0m'interface_list'\u001b[0m: \u001b[0m[\u001b[0m \u001b[0m\u001b[0m'Ethernet1'\u001b[0m,\n", + " \u001b[0m'Ethernet2'\u001b[0m,\n", + " \u001b[0m'Ethernet3'\u001b[0m,\n", + " \u001b[0m'Ethernet4'\u001b[0m,\n", + " \u001b[0m'Management1'\u001b[0m]\u001b[0m,\n", + " \u001b[0m'model'\u001b[0m: \u001b[0m'vEOS'\u001b[0m,\n", + " \u001b[0m'os_version'\u001b[0m: \u001b[0m'4.15.5M-3054042.4155M'\u001b[0m,\n", + " \u001b[0m'serial_number'\u001b[0m: \u001b[0m''\u001b[0m,\n", + " \u001b[0m'uptime'\u001b[0m: \u001b[0m'...'\u001b[0m,\n", + " \u001b[0m'vendor'\u001b[0m: \u001b[0m'Arista'\u001b[0m}\u001b[0m}\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m^^^^ END task_manages_connection_manually ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", + "\u001b[0m" + ] + } + ], + "source": [ + "def task_manages_connection_manually(task):\n", + " task.host.open_connection(\"napalm\")\n", + " r = task.run(\n", + " task=napalm_get,\n", + " getters=[\"facts\"]\n", + " )\n", + " task.host.close_connection(\"napalm\")\n", + " \n", + "nr = InitNornir(config_file=\"config.yaml\")\n", + "rtr = nr.filter(name=\"rtr00\")\n", + "r = rtr.run(\n", + " task=task_manages_connection_manually,\n", + ")\n", + "print_result(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Specifying connection parameters\n", + "\n", + "When using the [open_connection](../ref/api/inventory.rst#nornir.core.inventory.Host.open_connection) you can specify any parameters you want. If you don't, or if you let nornir open the connection automatically, nornir will read those parameters from the inventory. You can specify standard attributes at the object level if you want to reuse them across different connections or you can override them for each connection:\n", + "\n", + "```\n", + "my_host:\n", + " # standard parameters that will be reused across different connections\n", + " username: my_user\n", + " password: my_password\n", + " port: 22\n", + " platform: linux\n", + " napalm_options:\n", + " # standard parameters that will be used only for the napalm connection\n", + " # missing parameters will be read from the top level\n", + " port: 443\n", + " platform: eos\n", + " advanced_options:\n", + " # advanced options that are specific to this connection type\n", + " optional_args:\n", + " eos_autoComplete: true\n", + " netmiko_options:\n", + " platform: arista_eos\n", + " advanced_options:\n", + " global_delay: 2\n", + "```" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/howto/handling_connections/get_facts.1 b/docs/howto/handling_connections/get_facts.1 new file mode 100644 index 00000000..cde51ba6 --- /dev/null +++ b/docs/howto/handling_connections/get_facts.1 @@ -0,0 +1,16 @@ +{ + "fqdn": "localhost", + "hostname": "localhost", + "interface_list": [ + "Ethernet1", + "Ethernet2", + "Ethernet3", + "Ethernet4", + "Management1" + ], + "model": "vEOS", + "os_version": "4.15.5M-3054042.4155M", + "serial_number": "", + "uptime": "...", + "vendor": "Arista" +} diff --git a/docs/howto/inventory/groups.yaml b/docs/howto/inventory/groups.yaml new file mode 100644 index 00000000..dfdd45e6 --- /dev/null +++ b/docs/howto/inventory/groups.yaml @@ -0,0 +1,2 @@ +--- +defaults: {} diff --git a/docs/howto/inventory/hosts.yaml b/docs/howto/inventory/hosts.yaml new file mode 100644 index 00000000..5bbd3aa5 --- /dev/null +++ b/docs/howto/inventory/hosts.yaml @@ -0,0 +1,7 @@ +--- +rtr00: # Used for the "handling connections" how to + platform: mock + napalm_options: + advanced_options: + optional_args: + path: handling_connections From 773113a9bf69fc8f6402690037fd7f85a7e6577c Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 12 Aug 2018 14:21:18 +0200 Subject: [PATCH 029/109] test jupyter notebooks --- requirements-dev.txt | 1 + tox.ini | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0d18a6ba..0ebd8918 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,5 @@ decorator +nbval pytest pytest-cov pylama diff --git a/tox.ini b/tox.ini index 098195b7..3b0bd624 100644 --- a/tox.ini +++ b/tox.ini @@ -40,3 +40,11 @@ deps = basepython = python3.6 commands = mypy . + +[testenv:nbval] +deps = + -rrequirements-dev.txt + +basepython = python3.6 +commands = + pytest --nbval docs/howto From 29ba624a7eda165c361235ac8f87b5edc95267e6 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 12 Aug 2018 14:21:30 +0200 Subject: [PATCH 030/109] added upgrading notes --- docs/index.rst | 1 + docs/upgrading/1_to_2.rst | 15 +++++++++++++++ docs/upgrading/index.rst | 8 ++++++++ 3 files changed, 24 insertions(+) create mode 100644 docs/upgrading/1_to_2.rst create mode 100644 docs/upgrading/index.rst diff --git a/docs/index.rst b/docs/index.rst index a5f980d6..58ebd6d0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -61,6 +61,7 @@ Contents configuration/index plugins/index ref/index + upgrading/index Contribute diff --git a/docs/upgrading/1_to_2.rst b/docs/upgrading/1_to_2.rst new file mode 100644 index 00000000..c129672c --- /dev/null +++ b/docs/upgrading/1_to_2.rst @@ -0,0 +1,15 @@ +Upgrading to nornir 2.x from 1.x +================================ + +Changes in the inventory +------------------------ + +When specifying connection parameters, in nornir 1.x those parameters where specified with attributes like ``nornir_username``, ``nornir_password``, etc. All of those have been removed and now the only supported parameters are: + +* ``hostname`` +* ``username`` +* ``password`` +* ``port`` (which replaces both ``nornir_ssh_port`` and ``nornir_network_api_port``) +* ``platform`` (which replaces both ``os`` and ``network_operating_system``) + +You can check the following how to for more details on `how to <../howto/handling_connections.rst>`_ use these parameters. diff --git a/docs/upgrading/index.rst b/docs/upgrading/index.rst new file mode 100644 index 00000000..9fd800f8 --- /dev/null +++ b/docs/upgrading/index.rst @@ -0,0 +1,8 @@ +Notes when upgrading nornir +=========================== + +.. toctree:: + :maxdepth: 1 + :glob: + + * From e463e549b540e75cd16d912ac6266d6b7cd05f62 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 12 Aug 2018 14:23:43 +0200 Subject: [PATCH 031/109] enable nbval test in travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0c930391..79e093d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,8 @@ matrix: env: TOXENV=pylama - python: 3.6 env: TOXENV=mypy + - python: 3.6 + env: TOXENV=nbval install: - pip install tox tox-travis coveralls script: From 1c7a0af23b4db3d1330cbdccc15bbf244b5e4d5d Mon Sep 17 00:00:00 2001 From: David Barroso Date: Mon, 13 Aug 2018 10:41:16 +0200 Subject: [PATCH 032/109] address comments --- docs/howto/handling_connections.ipynb | 4 ++-- docs/howto/inventory/hosts.yaml | 2 +- nornir/core/connections.py | 2 +- nornir/core/inventory.py | 22 +++++++++---------- nornir/plugins/connections/napalm.py | 14 +++++------- nornir/plugins/connections/netmiko.py | 8 +++---- nornir/plugins/connections/paramiko.py | 10 ++++----- tests/core/test_connections.py | 8 +++---- tests/inventory_data/hosts.yaml | 2 +- .../tasks/networking/test_napalm_cli.py | 6 ++--- .../tasks/networking/test_napalm_configure.py | 12 +++++----- .../tasks/networking/test_napalm_get.py | 16 +++++++------- .../tasks/networking/test_napalm_validate.py | 10 ++++----- 13 files changed, 56 insertions(+), 60 deletions(-) diff --git a/docs/howto/handling_connections.ipynb b/docs/howto/handling_connections.ipynb index 005c2f3b..f0ec4bde 100644 --- a/docs/howto/handling_connections.ipynb +++ b/docs/howto/handling_connections.ipynb @@ -138,13 +138,13 @@ " # missing parameters will be read from the top level\n", " port: 443\n", " platform: eos\n", - " advanced_options:\n", + " connection_options:\n", " # advanced options that are specific to this connection type\n", " optional_args:\n", " eos_autoComplete: true\n", " netmiko_options:\n", " platform: arista_eos\n", - " advanced_options:\n", + " connection_options:\n", " global_delay: 2\n", "```" ] diff --git a/docs/howto/inventory/hosts.yaml b/docs/howto/inventory/hosts.yaml index 5bbd3aa5..2ef3b5ec 100644 --- a/docs/howto/inventory/hosts.yaml +++ b/docs/howto/inventory/hosts.yaml @@ -2,6 +2,6 @@ rtr00: # Used for the "handling connections" how to platform: mock napalm_options: - advanced_options: + connection_options: optional_args: path: handling_connections diff --git a/nornir/core/connections.py b/nornir/core/connections.py index e40b95c0..fe50090c 100644 --- a/nornir/core/connections.py +++ b/nornir/core/connections.py @@ -30,7 +30,7 @@ def open( password: Optional[str], port: Optional[int], platform: Optional[str], - advanced_options: Optional[Dict[str, Any]] = None, + connection_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: """ diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 1fff9f88..4e815fca 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -1,11 +1,11 @@ import getpass +from collections import Mapping from typing import Any, Dict, Optional from nornir.core.configuration import Config from nornir.core.connections import Connections from nornir.core.exceptions import ConnectionAlreadyOpen, ConnectionNotOpen -from ruamel.yaml.comments import CommentedMap VarsDict = Dict[str, Any] HostsDict = Dict[str, VarsDict] @@ -231,7 +231,7 @@ def get_connection_parameters( "username": self.username, "password": self.password, "platform": self.platform, - "advanced_options": {}, + "connection_options": {}, } else: conn_params = self.get(f"{connection}_options", {}) @@ -241,7 +241,7 @@ def get_connection_parameters( "username": conn_params.get("username", self.username), "password": conn_params.get("password", self.password), "platform": conn_params.get("platform", self.platform), - "advanced_options": conn_params.get("advanced_options", {}), + "connection_options": conn_params.get("connection_options", {}), } def get_connection(self, connection: str) -> Any: @@ -289,8 +289,8 @@ def open_connection( username: Optional[str] = None, password: Optional[str] = None, port: Optional[int] = None, - platform: Optional[int] = None, - advanced_options: Optional[int] = None, + platform: Optional[str] = None, + connection_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, default_to_host_attributes: bool = True, ) -> None: @@ -318,9 +318,9 @@ def open_connection( password=password if password is not None else conn_params["password"], port=port if port is not None else conn_params["port"], platform=platform if platform is not None else conn_params["platform"], - advanced_options=advanced_options - if advanced_options is not None - else conn_params["advanced_options"], + connection_options=connection_options + if connection_options is not None + else conn_params["connection_options"], configuration=configuration if configuration is not None else self.nornir.config, @@ -332,7 +332,7 @@ def open_connection( password=password, port=port, platform=platform, - advanced_options=advanced_options, + connection_options=connection_options, configuration=configuration, ) return self.connections[connection] @@ -389,7 +389,7 @@ def __init__( for group_name, group_details in groups.items(): if group_details is None: group = Group(name=group_name, nornir=nornir) - elif isinstance(group_details, (dict, CommentedMap)): + elif isinstance(group_details, Mapping): group = Group(name=group_name, nornir=nornir, **group_details) elif isinstance(group_details, Group): group = group_details @@ -407,7 +407,7 @@ def __init__( self.hosts = {} for n, h in hosts.items(): - if isinstance(h, (dict, CommentedMap)): + if isinstance(h, Mapping): h = Host(name=n, nornir=nornir, defaults=self.defaults, **h) if transform_function: diff --git a/nornir/plugins/connections/napalm.py b/nornir/plugins/connections/napalm.py index a1e86ea6..9c1360fb 100644 --- a/nornir/plugins/connections/napalm.py +++ b/nornir/plugins/connections/napalm.py @@ -12,8 +12,7 @@ class Napalm(ConnectionPlugin): relevant connection. Inventory: - advanced_options: passed as it is to the napalm driver - advanced_options["timeout"]: maps to ``timeout``. + connection_options: passed as it is to the napalm driver """ def open( @@ -23,10 +22,10 @@ def open( password: Optional[str], port: Optional[int], platform: Optional[str], - advanced_options: Optional[Dict[str, Any]] = None, + connection_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: - advanced_options = advanced_options or {} + connection_options = connection_options or {} parameters: Dict[str, Any] = { "hostname": hostname, @@ -34,14 +33,11 @@ def open( "password": password, "optional_args": {}, } - parameters.update(advanced_options) + parameters.update(connection_options) - if port and "port" not in advanced_options: + if port and "port" not in connection_options: parameters["optional_args"]["port"] = port - if advanced_options.get("timeout"): - parameters["timeout"] = advanced_options["timeout"] - network_driver = get_network_driver(platform) connection = network_driver(**parameters) connection.open() diff --git a/nornir/plugins/connections/netmiko.py b/nornir/plugins/connections/netmiko.py index cc16963f..8dfdc1d4 100644 --- a/nornir/plugins/connections/netmiko.py +++ b/nornir/plugins/connections/netmiko.py @@ -20,7 +20,7 @@ class Netmiko(ConnectionPlugin): relevant connection. Inventory: - advanced_options: maps to argument passed to ``ConnectHandler``. + connection_options: maps to argument passed to ``ConnectHandler``. """ def open( @@ -30,7 +30,7 @@ def open( password: Optional[str], port: Optional[int], platform: Optional[str], - advanced_options: Optional[Dict[str, Any]] = None, + connection_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: parameters = { @@ -45,8 +45,8 @@ def open( platform = napalm_to_netmiko_map.get(platform, platform) parameters["device_type"] = platform - advanced_options = advanced_options or {} - parameters.update(advanced_options) + connection_options = connection_options or {} + parameters.update(connection_options) self.connection = ConnectHandler(**parameters) def close(self) -> None: diff --git a/nornir/plugins/connections/paramiko.py b/nornir/plugins/connections/paramiko.py index a03779f6..abb1f49a 100644 --- a/nornir/plugins/connections/paramiko.py +++ b/nornir/plugins/connections/paramiko.py @@ -13,7 +13,7 @@ class Paramiko(ConnectionPlugin): relevant connection. Inventory: - advanced_options: maps to argument passed to ``ConnectHandler``. + connection_options: maps to argument passed to ``ConnectHandler``. """ def open( @@ -23,10 +23,10 @@ def open( password: Optional[str], port: Optional[int], platform: Optional[str], - advanced_options: Optional[Dict[str, Any]] = None, + connection_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: - advanced_options = advanced_options or {} + connection_options = connection_options or {} client = paramiko.SSHClient() client._policy = paramiko.WarningPolicy() @@ -60,8 +60,8 @@ def open( if "identityfile" in user_config: parameters["key_filename"] = user_config["identityfile"] - advanced_options.update(parameters) - client.connect(**advanced_options) + connection_options.update(parameters) + client.connect(**connection_options) self.connection = client def close(self) -> None: diff --git a/tests/core/test_connections.py b/tests/core/test_connections.py index 14341a9d..b400c147 100644 --- a/tests/core/test_connections.py +++ b/tests/core/test_connections.py @@ -13,7 +13,7 @@ def open( password: Optional[str], port: Optional[int], platform: Optional[str], - advanced_options: Optional[Dict[str, Any]] = None, + connection_options: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: self.connection = True @@ -23,7 +23,7 @@ def open( self.password = password self.port = port self.platform = platform - self.advanced_options = advanced_options + self.connection_options = connection_options self.configuration = configuration def close(self) -> None: @@ -105,7 +105,7 @@ def test_validate_params_simple(self, nornir): "password": "docker", "port": 65002, "platform": "junos", - "advanced_options": {}, + "connection_options": {}, } nr = nornir.filter(name="dev2.group_1") r = nr.run( @@ -125,7 +125,7 @@ def test_validate_params_overrides(self, nornir): "password": "docker", "port": None, "platform": "junos", - "advanced_options": {"awesome_feature": 1}, + "connection_options": {"awesome_feature": 1}, } nr = nornir.filter(name="dev2.group_1") r = nr.run(task=validate_params, conn="dummy", params=params, num_workers=1) diff --git a/tests/inventory_data/hosts.yaml b/tests/inventory_data/hosts.yaml index c2d6f46d..89f70112 100644 --- a/tests/inventory_data/hosts.yaml +++ b/tests/inventory_data/hosts.yaml @@ -29,7 +29,7 @@ dev2.group_1: dummy_options: hostname: overriden_hostname port: null - advanced_options: + connection_options: awesome_feature: 1 dev3.group_2: diff --git a/tests/plugins/tasks/networking/test_napalm_cli.py b/tests/plugins/tasks/networking/test_napalm_cli.py index d508e680..2dffff4f 100644 --- a/tests/plugins/tasks/networking/test_napalm_cli.py +++ b/tests/plugins/tasks/networking/test_napalm_cli.py @@ -11,12 +11,12 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__)) + "/mocked/napalm_cli" -def connect(task, advanced_options): +def connect(task, connection_options): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( "napalm", - advanced_options={"optional_args": advanced_options}, + connection_options={"optional_args": connection_options}, default_to_host_attributes=True, ) @@ -25,7 +25,7 @@ class Test(object): def test_napalm_cli(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_cli"} d = nornir.filter(name="dev3.group_2") - d.run(connect, advanced_options=opt) + d.run(connect, connection_options=opt) result = d.run( networking.napalm_cli, commands=["show version", "show interfaces"] ) diff --git a/tests/plugins/tasks/networking/test_napalm_configure.py b/tests/plugins/tasks/networking/test_napalm_configure.py index 1260ab82..97bed5e3 100644 --- a/tests/plugins/tasks/networking/test_napalm_configure.py +++ b/tests/plugins/tasks/networking/test_napalm_configure.py @@ -8,12 +8,12 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__)) + "/mocked/napalm_configure" -def connect(task, advanced_options): +def connect(task, connection_options): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( "napalm", - advanced_options={"optional_args": advanced_options}, + connection_options={"optional_args": connection_options}, default_to_host_attributes=True, ) @@ -23,7 +23,7 @@ def test_napalm_configure_change_dry_run(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_configure_change_dry_run"} configuration = "hostname changed-hostname" d = nornir.filter(name="dev3.group_2") - d.run(connect, advanced_options=opt) + d.run(connect, connection_options=opt) result = d.run(networking.napalm_configure, configuration=configuration) assert result for h, r in result.items(): @@ -34,7 +34,7 @@ def test_napalm_configure_change_commit(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_configure_change_commit/step1"} configuration = "hostname changed-hostname" d = nornir.filter(name="dev3.group_2") - d.run(connect, advanced_options=opt) + d.run(connect, connection_options=opt) result = d.run( networking.napalm_configure, dry_run=False, configuration=configuration ) @@ -43,7 +43,7 @@ def test_napalm_configure_change_commit(self, nornir): assert "+hostname changed-hostname" in r.diff assert r.changed opt = {"path": THIS_DIR + "/test_napalm_configure_change_commit/step2"} - d.run(connect, advanced_options=opt) + d.run(connect, connection_options=opt) result = d.run( networking.napalm_configure, dry_run=True, configuration=configuration ) @@ -57,7 +57,7 @@ def test_napalm_configure_change_error(self, nornir): configuration = "hostname changed_hostname" d = nornir.filter(name="dev3.group_2") - d.run(connect, advanced_options=opt) + d.run(connect, connection_options=opt) results = d.run(networking.napalm_configure, configuration=configuration) processed = False for result in results.values(): diff --git a/tests/plugins/tasks/networking/test_napalm_get.py b/tests/plugins/tasks/networking/test_napalm_get.py index 191ec14a..80389d4d 100644 --- a/tests/plugins/tasks/networking/test_napalm_get.py +++ b/tests/plugins/tasks/networking/test_napalm_get.py @@ -6,12 +6,12 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__)) + "/mocked/napalm_get" -def connect(task, advanced_options): +def connect(task, connection_options): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( "napalm", - advanced_options={"optional_args": advanced_options}, + connection_options={"optional_args": connection_options}, default_to_host_attributes=True, ) @@ -20,7 +20,7 @@ class Test(object): def test_napalm_getters(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters"} d = nornir.filter(name="dev3.group_2") - d.run(task=connect, advanced_options=opt) + d.run(task=connect, connection_options=opt) result = d.run(networking.napalm_get, getters=["facts", "interfaces"]) assert result for h, r in result.items(): @@ -30,7 +30,7 @@ def test_napalm_getters(self, nornir): def test_napalm_getters_error(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_error"} d = nornir.filter(name="dev3.group_2") - d.run(task=connect, advanced_options=opt) + d.run(task=connect, connection_options=opt) results = d.run(networking.napalm_get, getters=["facts", "interfaces"]) processed = False @@ -43,7 +43,7 @@ def test_napalm_getters_error(self, nornir): def test_napalm_getters_with_options_error(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_single_with_options"} d = nornir.filter(name="dev3.group_2") - d.run(task=connect, advanced_options=opt) + d.run(task=connect, connection_options=opt) result = d.run( task=networking.napalm_get, getters=["config"], nonexistent="asdsa" ) @@ -56,7 +56,7 @@ def test_napalm_getters_with_options_error(self, nornir): def test_napalm_getters_with_options_error_optional_args(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_single_with_options"} d = nornir.filter(name="dev3.group_2") - d.run(task=connect, advanced_options=opt) + d.run(task=connect, connection_options=opt) result = d.run( task=networking.napalm_get, getters=["config"], @@ -71,7 +71,7 @@ def test_napalm_getters_with_options_error_optional_args(self, nornir): def test_napalm_getters_single_with_options(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_single_with_options"} d = nornir.filter(name="dev3.group_2") - d.run(task=connect, advanced_options=opt) + d.run(task=connect, connection_options=opt) result = d.run( task=networking.napalm_get, getters=["config"], retrieve="candidate" ) @@ -83,7 +83,7 @@ def test_napalm_getters_single_with_options(self, nornir): def test_napalm_getters_multiple_with_options(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_multiple_with_options"} d = nornir.filter(name="dev3.group_2") - d.run(task=connect, advanced_options=opt) + d.run(task=connect, connection_options=opt) result = d.run( task=networking.napalm_get, getters=["config", "facts"], diff --git a/tests/plugins/tasks/networking/test_napalm_validate.py b/tests/plugins/tasks/networking/test_napalm_validate.py index e30cb285..8dd85552 100644 --- a/tests/plugins/tasks/networking/test_napalm_validate.py +++ b/tests/plugins/tasks/networking/test_napalm_validate.py @@ -6,12 +6,12 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__)) -def connect(task, advanced_options): +def connect(task, connection_options): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( "napalm", - advanced_options={"optional_args": advanced_options}, + connection_options={"optional_args": connection_options}, default_to_host_attributes=True, ) @@ -20,7 +20,7 @@ class Test(object): def test_napalm_validate_src_ok(self, nornir): opt = {"path": THIS_DIR + "/mocked/napalm_get/test_napalm_getters"} d = nornir.filter(name="dev3.group_2") - d.run(connect, advanced_options=opt) + d.run(connect, connection_options=opt) result = d.run( networking.napalm_validate, src=THIS_DIR + "/data/validate_ok.yaml" ) @@ -31,7 +31,7 @@ def test_napalm_validate_src_ok(self, nornir): def test_napalm_validate_src_error(self, nornir): opt = {"path": THIS_DIR + "/mocked/napalm_get/test_napalm_getters"} d = nornir.filter(name="dev3.group_2") - d.run(connect, advanced_options=opt) + d.run(connect, connection_options=opt) result = d.run( networking.napalm_validate, src=THIS_DIR + "/data/validate_error.yaml" @@ -44,7 +44,7 @@ def test_napalm_validate_src_error(self, nornir): def test_napalm_validate_src_validate_source(self, nornir): opt = {"path": THIS_DIR + "/mocked/napalm_get/test_napalm_getters"} d = nornir.filter(name="dev3.group_2") - d.run(connect, advanced_options=opt) + d.run(connect, connection_options=opt) validation_dict = [{"get_interfaces": {"Ethernet1": {"description": ""}}}] From 72d664f7a06f5b7fee02db2a8f361d5308f7d9e9 Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 13 Aug 2018 03:01:37 -0700 Subject: [PATCH 033/109] Patch connection close (#218) * Fix close_connections for loop issue * Minor comment change --- nornir/core/inventory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 4e815fca..aa03b64a 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -345,7 +345,9 @@ def close_connection(self, connection: str) -> None: self.connections.pop(connection).close() def close_connections(self) -> None: - for connection in self.connections: + # Decouple deleting dictionary elements from iterating over connections dict + existing_conns = list(self.connections.keys()) + for connection in existing_conns: self.close_connection(connection) From a4126660d6f963d9cb2f0a475f1f480ad97ee43d Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 13 Aug 2018 23:50:49 -0700 Subject: [PATCH 034/109] Fix napalm plugin port reference (#228) --- nornir/plugins/connections/napalm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nornir/plugins/connections/napalm.py b/nornir/plugins/connections/napalm.py index 9c1360fb..56090d40 100644 --- a/nornir/plugins/connections/napalm.py +++ b/nornir/plugins/connections/napalm.py @@ -35,7 +35,7 @@ def open( } parameters.update(connection_options) - if port and "port" not in connection_options: + if port and "port" not in connection_options["optional_args"]: parameters["optional_args"]["port"] = port network_driver = get_network_driver(platform) From f48980830c7d5bf3f3a529845d38630c93006721 Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 13 Aug 2018 23:51:32 -0700 Subject: [PATCH 035/109] Add setters for first-level Nornir inventory attributes (#229) * Add support for setters * black * Minor cleanup --- nornir/core/inventory.py | 20 ++++++++++++++++++++ tests/core/test_inventory.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index aa03b64a..aaba2b2c 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -201,26 +201,46 @@ def hostname(self): """String used to connect to the device. Either ``hostname`` or ``self.name``""" return self.get("hostname", self.name) + @hostname.setter + def hostname(self, value): + self.data["hostname"] = value + @property def port(self): """Either ``port`` or ``None``.""" return self.get("port") + @port.setter + def port(self, value): + self.data["port"] = value + @property def username(self): """Either ``username`` or user running the script.""" return self.get("username", getpass.getuser()) + @username.setter + def username(self, value): + self.data["username"] = value + @property def password(self): """Either ``password`` or empty string.""" return self.get("password", "") + @password.setter + def password(self, value): + self.data["password"] = value + @property def platform(self): """OS the device is running. Defaults to ``platform``.""" return self.get("platform") + @platform.setter + def platform(self, value): + self.data["platform"] = value + def get_connection_parameters( self, connection: Optional[str] = None ) -> Dict[str, Any]: diff --git a/tests/core/test_inventory.py b/tests/core/test_inventory.py index ebb9ef2a..03dfd87d 100644 --- a/tests/core/test_inventory.py +++ b/tests/core/test_inventory.py @@ -249,3 +249,31 @@ def test_to_dict(self): }, } assert inventory.filter(role="www").to_dict() == expected + + def test_setters(self): + """Test explicit setters specified in inventory.""" + defaults = {} + g1 = Group(name="g1") + h1 = Host(name="host1", groups=[g1], defaults=defaults) + + g1.hostname = "group_hostname" + assert h1.hostname == "group_hostname" + g1.platform = "group_platform" + assert h1.platform == "group_platform" + g1.username = "group_username" + assert h1.username == "group_username" + g1.password = "group_password" + assert h1.password == "group_password" + g1.port = 9999 + assert h1.port == 9999 + + h1.hostname = "alt_hostname" + assert h1.hostname == "alt_hostname" + h1.platform = "alt_platform" + assert h1.platform == "alt_platform" + h1.username = "alt_username" + assert h1.username == "alt_username" + h1.password = "alt_password" + assert h1.password == "alt_password" + h1.port = 9998 + assert h1.port == 9998 From de9eba186f45a37af5ff56e7f6dc505d72842b02 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Thu, 16 Aug 2018 13:10:58 +0200 Subject: [PATCH 036/109] Migrate how tos to testable notebooks (#227) * we were failing to keep a reference to parent mutable * migrate tutorial to a testable playbook * rearrange for clarity * rearrange howtos slightly * added advanced_filtering tutorial --- docs/howto/advanced_filtering.ipynb | 352 ++++++++++++++++++ docs/howto/advanced_filtering/config.yaml | 6 + .../advanced_filtering/inventory/groups.yaml | 16 + .../advanced_filtering/inventory/hosts.yaml | 72 ++++ docs/howto/config.yaml | 6 - docs/howto/handling_connections.ipynb | 4 +- docs/howto/handling_connections/config.yaml | 6 + .../inventory/groups.yaml | 0 .../inventory/hosts.yaml | 4 +- .../{ => mocked_data}/get_facts.1 | 0 docs/howto/transforming_inventory_data.ipynb | 281 ++++++++++++++ docs/howto/transforming_inventory_data.rst | 25 -- .../transforming_inventory_data/config.yaml | 6 + .../transforming_inventory_data/helpers.py | 3 + .../inventory/groups.yaml | 3 + .../inventory/hosts.yaml | 5 + docs/howto/writing_a_connection_plugin.rst | 4 - docs/howto/writing_a_custom_inventory.rst | 58 --- .../writing_a_custom_inventory_plugin.ipynb | 125 +++++++ .../my_inventory.py | 33 ++ nornir/core/inventory.py | 4 +- 21 files changed, 914 insertions(+), 99 deletions(-) create mode 100644 docs/howto/advanced_filtering.ipynb create mode 100644 docs/howto/advanced_filtering/config.yaml create mode 100644 docs/howto/advanced_filtering/inventory/groups.yaml create mode 100644 docs/howto/advanced_filtering/inventory/hosts.yaml delete mode 100644 docs/howto/config.yaml create mode 100644 docs/howto/handling_connections/config.yaml rename docs/howto/{ => handling_connections}/inventory/groups.yaml (100%) rename docs/howto/{ => handling_connections}/inventory/hosts.yaml (50%) rename docs/howto/handling_connections/{ => mocked_data}/get_facts.1 (100%) create mode 100644 docs/howto/transforming_inventory_data.ipynb delete mode 100644 docs/howto/transforming_inventory_data.rst create mode 100644 docs/howto/transforming_inventory_data/config.yaml create mode 100644 docs/howto/transforming_inventory_data/helpers.py create mode 100644 docs/howto/transforming_inventory_data/inventory/groups.yaml create mode 100644 docs/howto/transforming_inventory_data/inventory/hosts.yaml delete mode 100644 docs/howto/writing_a_connection_plugin.rst delete mode 100644 docs/howto/writing_a_custom_inventory.rst create mode 100644 docs/howto/writing_a_custom_inventory_plugin.ipynb create mode 100644 docs/howto/writing_a_custom_inventory_plugin/my_inventory.py diff --git a/docs/howto/advanced_filtering.ipynb b/docs/howto/advanced_filtering.ipynb new file mode 100644 index 00000000..de838f99 --- /dev/null +++ b/docs/howto/advanced_filtering.ipynb @@ -0,0 +1,352 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Advanced filtering\n", + "\n", + "In this tutorial we are going to see how to use the ``F`` object to do advanced filtering of hosts. Let's start by initiating nornir and looking at the inventory:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from nornir.core import InitNornir\n", + "from nornir.core.filter import F\n", + "\n", + "nr = InitNornir(config_file=\"advanced_filtering/config.yaml\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---\r\n", + "cat:\r\n", + " domestic: true\r\n", + " diet: omnivore\r\n", + " groups:\r\n", + " - terrestrial\r\n", + " - mammal\r\n", + " additional_data:\r\n", + " lifespan: 17\r\n", + " famous_members:\r\n", + " - garfield\r\n", + " - felix\r\n", + " - grumpy\r\n", + "\r\n", + "bat:\r\n", + " domestic: false\r\n", + " fly: true\r\n", + " diet: carnivore\r\n", + " groups:\r\n", + " - terrestrial\r\n", + " - mammal\r\n", + " additional_data:\r\n", + " lifespan: 15\r\n", + " famous_members:\r\n", + " - batman\r\n", + " - count chocula\r\n", + " - nosferatu\r\n", + "\r\n", + "eagle:\r\n", + " domestic: false\r\n", + " diet: carnivore\r\n", + " groups:\r\n", + " - terrestrial\r\n", + " - bird\r\n", + " additional_data:\r\n", + " lifespan: 50\r\n", + " famous_members:\r\n", + " - thorondor\r\n", + " - sam\r\n", + "\r\n", + "canary:\r\n", + " domestic: true\r\n", + " diet: herbivore\r\n", + " groups:\r\n", + " - terrestrial\r\n", + " - bird\r\n", + " additional_data:\r\n", + " lifespan: 15\r\n", + " famous_members:\r\n", + " - tweetie\r\n", + "\r\n", + "caterpillaer:\r\n", + " domestic: false\r\n", + " diet: herbivore\r\n", + " groups:\r\n", + " - terrestrial\r\n", + " - invertebrate\r\n", + " additional_data:\r\n", + " lifespan: 1\r\n", + " famous_members:\r\n", + " - Hookah-Smoking\r\n", + "\r\n", + "octopus:\r\n", + " domestic: false\r\n", + " diet: carnivore\r\n", + " groups:\r\n", + " - marine\r\n", + " - invertebrate\r\n", + " additional_data:\r\n", + " lifespan: 1\r\n", + " famous_members:\r\n", + " - sharktopus\r\n" + ] + } + ], + "source": [ + "%cat advanced_filtering/inventory/hosts.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---\r\n", + "defaults: {}\r\n", + "mammal:\r\n", + " reproduction: birth\r\n", + " fly: false\r\n", + "\r\n", + "bird:\r\n", + " reproduction: eggs\r\n", + " fly: true\r\n", + "\r\n", + "invertebrate:\r\n", + " reproduction: mitosis\r\n", + " fly: false\r\n", + "\r\n", + "terrestrial: {}\r\n", + "marine: {}\r\n" + ] + } + ], + "source": [ + "%cat advanced_filtering/inventory/groups.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As you can see we have built ourselves a collection of animals with different properties. The ``F`` object let's you access the magic methods of each typesby just prepeding two underscores and the the name of the magic method. For instance, if you want to check if a list contains a particular element you can just prepend ``__contains``. Let's use this feature to retrieve all the animals that belong to the group ``bird``:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['eagle', 'canary'])\n" + ] + } + ], + "source": [ + "birds = nr.filter(F(groups__contains=\"bird\"))\n", + "print(birds.inventory.hosts.keys())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can also invert the ``F`` object by prepending ``~``:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['cat', 'bat', 'caterpillaer', 'octopus'])\n" + ] + } + ], + "source": [ + "not_birds = nr.filter(~F(groups__contains=\"bird\"))\n", + "print(not_birds.inventory.hosts.keys())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can also combine ``F`` objects and perform AND and OR operations with the symbols ``&`` and ``|`` (pipe) respectively:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['cat', 'eagle', 'canary'])\n" + ] + } + ], + "source": [ + "domestic_or_bird = nr.filter(F(groups__contains=\"bird\") | F(domestic=True))\n", + "print(domestic_or_bird.inventory.hosts.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['cat'])\n" + ] + } + ], + "source": [ + "domestic_mammals = nr.filter(F(groups__contains=\"mammal\") & F(domestic=True))\n", + "print(domestic_mammals.inventory.hosts.keys())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As expected, you can combine all of the symbols:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['canary'])\n" + ] + } + ], + "source": [ + "flying_not_carnivore = nr.filter(F(fly=True) & ~F(diet=\"carnivore\"))\n", + "print(flying_not_carnivore.inventory.hosts.keys())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also access nested data the same way you access magic methods, by appending two underscores and the data you want to access. You can keep building on this as much as needed and even access the magic methods of the nested data. For instance, let's get the animals that have a lifespan greater or equal than 15:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['cat', 'bat', 'eagle', 'canary'])\n" + ] + } + ], + "source": [ + "long_lived = nr.filter(F(additional_data__lifespan__ge=15))\n", + "print(long_lived.inventory.hosts.keys())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "There are two extra facilities to help you working with lists; ``any`` and ``all``. Those facilities let's you send a list of elements and get the objects that has either any of the members or all of them. For instance:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['octopus'])\n" + ] + } + ], + "source": [ + "marine_and_invertebrates = nr.filter(F(groups__all=[\"marine\", \"invertebrate\"]))\n", + "print(marine_and_invertebrates.inventory.hosts.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['eagle', 'canary', 'caterpillaer', 'octopus'])\n" + ] + } + ], + "source": [ + "bird_or_invertebrates = nr.filter(F(groups__any=[\"bird\", \"invertebrate\"]))\n", + "print(bird_or_invertebrates.inventory.hosts.keys())" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/howto/advanced_filtering/config.yaml b/docs/howto/advanced_filtering/config.yaml new file mode 100644 index 00000000..fdefe2a6 --- /dev/null +++ b/docs/howto/advanced_filtering/config.yaml @@ -0,0 +1,6 @@ +--- +inventory: nornir.plugins.inventory.simple.SimpleInventory +SimpleInventory: + host_file: "advanced_filtering/inventory/hosts.yaml" + group_file: "advanced_filtering/inventory/groups.yaml" + diff --git a/docs/howto/advanced_filtering/inventory/groups.yaml b/docs/howto/advanced_filtering/inventory/groups.yaml new file mode 100644 index 00000000..60f4fbf8 --- /dev/null +++ b/docs/howto/advanced_filtering/inventory/groups.yaml @@ -0,0 +1,16 @@ +--- +defaults: {} +mammal: + reproduction: birth + fly: false + +bird: + reproduction: eggs + fly: true + +invertebrate: + reproduction: mitosis + fly: false + +terrestrial: {} +marine: {} diff --git a/docs/howto/advanced_filtering/inventory/hosts.yaml b/docs/howto/advanced_filtering/inventory/hosts.yaml new file mode 100644 index 00000000..48fdd6de --- /dev/null +++ b/docs/howto/advanced_filtering/inventory/hosts.yaml @@ -0,0 +1,72 @@ +--- +cat: + domestic: true + diet: omnivore + groups: + - terrestrial + - mammal + additional_data: + lifespan: 17 + famous_members: + - garfield + - felix + - grumpy + +bat: + domestic: false + fly: true + diet: carnivore + groups: + - terrestrial + - mammal + additional_data: + lifespan: 15 + famous_members: + - batman + - count chocula + - nosferatu + +eagle: + domestic: false + diet: carnivore + groups: + - terrestrial + - bird + additional_data: + lifespan: 50 + famous_members: + - thorondor + - sam + +canary: + domestic: true + diet: herbivore + groups: + - terrestrial + - bird + additional_data: + lifespan: 15 + famous_members: + - tweetie + +caterpillaer: + domestic: false + diet: herbivore + groups: + - terrestrial + - invertebrate + additional_data: + lifespan: 1 + famous_members: + - Hookah-Smoking + +octopus: + domestic: false + diet: carnivore + groups: + - marine + - invertebrate + additional_data: + lifespan: 1 + famous_members: + - sharktopus diff --git a/docs/howto/config.yaml b/docs/howto/config.yaml deleted file mode 100644 index ae8ee1ea..00000000 --- a/docs/howto/config.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -inventory: nornir.plugins.inventory.simple.SimpleInventory -SimpleInventory: - host_file: "inventory/hosts.yaml" - group_file: "inventory/groups.yaml" - diff --git a/docs/howto/handling_connections.ipynb b/docs/howto/handling_connections.ipynb index f0ec4bde..d8912a2f 100644 --- a/docs/howto/handling_connections.ipynb +++ b/docs/howto/handling_connections.ipynb @@ -52,7 +52,7 @@ } ], "source": [ - "nr = InitNornir(config_file=\"config.yaml\")\n", + "nr = InitNornir(config_file=\"handling_connections/config.yaml\")\n", "rtr = nr.filter(name=\"rtr00\")\n", "r = rtr.run(\n", " task=napalm_get,\n", @@ -110,7 +110,7 @@ " )\n", " task.host.close_connection(\"napalm\")\n", " \n", - "nr = InitNornir(config_file=\"config.yaml\")\n", + "nr = InitNornir(config_file=\"handling_connections/config.yaml\")\n", "rtr = nr.filter(name=\"rtr00\")\n", "r = rtr.run(\n", " task=task_manages_connection_manually,\n", diff --git a/docs/howto/handling_connections/config.yaml b/docs/howto/handling_connections/config.yaml new file mode 100644 index 00000000..19d67fcb --- /dev/null +++ b/docs/howto/handling_connections/config.yaml @@ -0,0 +1,6 @@ +--- +inventory: nornir.plugins.inventory.simple.SimpleInventory +SimpleInventory: + host_file: "handling_connections/inventory/hosts.yaml" + group_file: "handling_connections/inventory/groups.yaml" + diff --git a/docs/howto/inventory/groups.yaml b/docs/howto/handling_connections/inventory/groups.yaml similarity index 100% rename from docs/howto/inventory/groups.yaml rename to docs/howto/handling_connections/inventory/groups.yaml diff --git a/docs/howto/inventory/hosts.yaml b/docs/howto/handling_connections/inventory/hosts.yaml similarity index 50% rename from docs/howto/inventory/hosts.yaml rename to docs/howto/handling_connections/inventory/hosts.yaml index 2ef3b5ec..5820cdd5 100644 --- a/docs/howto/inventory/hosts.yaml +++ b/docs/howto/handling_connections/inventory/hosts.yaml @@ -1,7 +1,7 @@ --- -rtr00: # Used for the "handling connections" how to +rtr00: platform: mock napalm_options: connection_options: optional_args: - path: handling_connections + path: handling_connections/mocked_data diff --git a/docs/howto/handling_connections/get_facts.1 b/docs/howto/handling_connections/mocked_data/get_facts.1 similarity index 100% rename from docs/howto/handling_connections/get_facts.1 rename to docs/howto/handling_connections/mocked_data/get_facts.1 diff --git a/docs/howto/transforming_inventory_data.ipynb b/docs/howto/transforming_inventory_data.ipynb new file mode 100644 index 00000000..b08a9e49 --- /dev/null +++ b/docs/howto/transforming_inventory_data.ipynb @@ -0,0 +1,281 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Transforming Inventory Data\n", + "\n", + "Imagine your data looks like:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---\r\n", + "rtr00:\r\n", + " user: automation_user\r\n", + "rtr01:\r\n", + " user: automation_user\r\n" + ] + } + ], + "source": [ + "%cat transforming_inventory_data/inventory/hosts.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And we want to do two things:\n", + "\n", + "1. Map ``user`` to ``username``\n", + "2. Prompt the user for the password and add it\n", + "\n", + "You can easily do that using a transform_function. For instance:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Modifying hosts' data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can modify inventory data (regardless of the plugin you are using) on the fly easily by password a ``transform_function`` like this:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'groups': {'defaults': ordereddict([('a_default_attribute', 'my_default')])},\n", + " 'hosts': {'rtr00': {'name': 'rtr00',\n", + " 'user': 'automation_user',\n", + " 'username': 'automation_user'},\n", + " 'rtr01': {'name': 'rtr01',\n", + " 'user': 'automation_user',\n", + " 'username': 'automation_user'}}}\n" + ] + } + ], + "source": [ + "from nornir.core import InitNornir\n", + "import pprint\n", + "\n", + "def adapt_host_data(host):\n", + " # This function receives a Host object for manipulation\n", + " host[\"username\"] = host[\"user\"]\n", + "\n", + "nr = InitNornir(\n", + " config_file=\"transforming_inventory_data/config.yaml\",\n", + " transform_function=adapt_host_data,\n", + ")\n", + "pprint.pprint(nr.inventory.to_dict())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Specifying the transform function in the configuration\n", + "\n", + "You can also specify the transform function in the configuration. In order for that to work the function must be importable. For instance:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "def adapt_host_data(host):\r\n", + " # This function receives a Host object for manipulation\r\n", + " host[\"username\"] = host[\"user\"]\r\n" + ] + } + ], + "source": [ + "%cat transforming_inventory_data/helpers.py" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's verify we can, indeed, import the function:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from transforming_inventory_data.helpers import adapt_host_data\n", + "adapt_host_data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now everything we have to do is put the import path as the ``transform_function`` configuration option:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---\r\n", + "inventory: nornir.plugins.inventory.simple.SimpleInventory\r\n", + "SimpleInventory:\r\n", + " host_file: \"transforming_inventory_data/inventory/hosts.yaml\"\r\n", + " group_file: \"transforming_inventory_data/inventory/groups.yaml\"\r\n", + "transform_function: \"transforming_inventory_data.helpers.adapt_host_data\"\r\n" + ] + } + ], + "source": [ + "%cat transforming_inventory_data/config.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And then we won't need to specify it when calling ``InitNornir``:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'groups': {'defaults': ordereddict([('a_default_attribute', 'my_default')])},\n", + " 'hosts': {'rtr00': {'name': 'rtr00',\n", + " 'user': 'automation_user',\n", + " 'username': 'automation_user'},\n", + " 'rtr01': {'name': 'rtr01',\n", + " 'user': 'automation_user',\n", + " 'username': 'automation_user'}}}\n" + ] + } + ], + "source": [ + "from nornir.core import InitNornir\n", + "import pprint\n", + "\n", + "nr = InitNornir(\n", + " config_file=\"transforming_inventory_data/config.yaml\",\n", + ")\n", + "pprint.pprint(nr.inventory.to_dict())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setting a default password\n", + "\n", + "You might be in a situation where you may want to prompt the user for some information, for instance, a password. You can leverage the ``defaults`` for something like this (although you could just put in under the hosts themselves or the groups). Let's see an example:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Before setting password: \n", + "After setting password: a_secret_password\n" + ] + } + ], + "source": [ + "from nornir.core import InitNornir\n", + "\n", + "# let's pretend we used raw_input or something like that\n", + "# password = raw_input(\"Please, enter password: \")\n", + "password = \"a_secret_password\"\n", + "\n", + "nr = InitNornir(\n", + " config_file=\"transforming_inventory_data/config.yaml\",\n", + ")\n", + "print(\"Before setting password: \", nr.inventory.hosts[\"rtr00\"].password)\n", + "nr.inventory.defaults[\"password\"] = password\n", + "print(\"After setting password: \", nr.inventory.hosts[\"rtr00\"][\"password\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For more information on how inheritance works check the tutorial section [inheritance model](../tutorials/intro/inventory.rst)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/howto/transforming_inventory_data.rst b/docs/howto/transforming_inventory_data.rst deleted file mode 100644 index fe5cf69e..00000000 --- a/docs/howto/transforming_inventory_data.rst +++ /dev/null @@ -1,25 +0,0 @@ -Transforming Inventory Data -=========================== - -Imagine your data looks like:: - - host1: - username: my_user - password: my_password - host2: - username: my_user - password: my_password - -It turns out nornir is going to look for ``nornir_username`` and ``nornir_password`` to use as credentials. You may not want to change the data in your backend and you may not want to write a custom inventory plugin just to accommodate this difference. Fortunately, ``nornir`` has you covered. You can write a function to do all the data manipulations you want and pass it to any inventory plugin. For instance:: - - def adapt_host_data(host): - host.data["nornir_username"] = host.data["username"] - host.data["nornir_password"] = host.data["password"] - - - inv = NSOTInventory(transform_function=adapt_host_data) - nornir = Nornir(inventory=inv) - -What's going to happen is that the inventory is going to create the :obj:`nornir.core.inventory.Host` and :obj:`nornir.core.inventory.Group` objects as usual and then finally the ``transform_function`` is going to be called for each individual host one by one. - -.. note:: This was a very simple example but the ``transform_function`` can basically do anything you want/need. diff --git a/docs/howto/transforming_inventory_data/config.yaml b/docs/howto/transforming_inventory_data/config.yaml new file mode 100644 index 00000000..8c8365e8 --- /dev/null +++ b/docs/howto/transforming_inventory_data/config.yaml @@ -0,0 +1,6 @@ +--- +inventory: nornir.plugins.inventory.simple.SimpleInventory +SimpleInventory: + host_file: "transforming_inventory_data/inventory/hosts.yaml" + group_file: "transforming_inventory_data/inventory/groups.yaml" +transform_function: "transforming_inventory_data.helpers.adapt_host_data" diff --git a/docs/howto/transforming_inventory_data/helpers.py b/docs/howto/transforming_inventory_data/helpers.py new file mode 100644 index 00000000..4642cfa9 --- /dev/null +++ b/docs/howto/transforming_inventory_data/helpers.py @@ -0,0 +1,3 @@ +def adapt_host_data(host): + # This function receives a Host object for manipulation + host["username"] = host["user"] diff --git a/docs/howto/transforming_inventory_data/inventory/groups.yaml b/docs/howto/transforming_inventory_data/inventory/groups.yaml new file mode 100644 index 00000000..fe974dda --- /dev/null +++ b/docs/howto/transforming_inventory_data/inventory/groups.yaml @@ -0,0 +1,3 @@ +--- +defaults: + a_default_attribute: my_default diff --git a/docs/howto/transforming_inventory_data/inventory/hosts.yaml b/docs/howto/transforming_inventory_data/inventory/hosts.yaml new file mode 100644 index 00000000..2e49ed07 --- /dev/null +++ b/docs/howto/transforming_inventory_data/inventory/hosts.yaml @@ -0,0 +1,5 @@ +--- +rtr00: + user: automation_user +rtr01: + user: automation_user diff --git a/docs/howto/writing_a_connection_plugin.rst b/docs/howto/writing_a_connection_plugin.rst deleted file mode 100644 index 12268c7f..00000000 --- a/docs/howto/writing_a_connection_plugin.rst +++ /dev/null @@ -1,4 +0,0 @@ -Writing a connection plugin -########################### - -See :obj:`nornir.core.connections.ConnectionPlugin` and `this `_. diff --git a/docs/howto/writing_a_custom_inventory.rst b/docs/howto/writing_a_custom_inventory.rst deleted file mode 100644 index 6ff73c4c..00000000 --- a/docs/howto/writing_a_custom_inventory.rst +++ /dev/null @@ -1,58 +0,0 @@ -Writing a custom inventory -========================== - -If you have your own backend with host information or you don't like the provided ones you can write your own custom inventory. Doing so is quite easy. A continuation you can find a very simple one with static data. - -.. code-block:: python - - from builtins import super - - from nornir.core.inventory import Inventory - - - class MyInventory(Inventory): - - def __init__(self, **kwargs): - # code to get the data - hosts = { - "host1": { - "data1": "value1", - "data2": "value2", - "groups": ["my_group1"], - }, - "host2": { - "data1": "value1", - "data2": "value2", - "groups": ["my_group1"], - } - } - groups = { - "my_group1": { - "more_data1": "more_value1", - "more_data2": "more_value2", - } - } - defaults = { - "location": "internet", - "language": "Python", - } - - # passing the data to the parent class so the data is - # transformed into actual Host/Group objects - # and set default data for all hosts - super().__init__(hosts, groups, defaults, **kwargs) - - -So if you want to make it dynamic everything you have to do is get the data yourself and organize it in a similar format to the one described in the example above. - -.. note:: it is not mandatory to use groups or defaults. Feel free to skip the attribute ``groups`` and just pass and empty dict or ``None`` to ``super()``. - -Finally, to have nornir use it, you can: - -.. code-block:: python - - from nornir.core import InitNornir - nr = InitNornir(inventory=MyInventory) - - -And that's it, you now have your own inventory plugin :) diff --git a/docs/howto/writing_a_custom_inventory_plugin.ipynb b/docs/howto/writing_a_custom_inventory_plugin.ipynb new file mode 100644 index 00000000..e3316d69 --- /dev/null +++ b/docs/howto/writing_a_custom_inventory_plugin.ipynb @@ -0,0 +1,125 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Writing a custom inventory plugin\n", + "\n", + "If you have your own backend with host information or you don't like the provided ones you can write your own custom inventory. Doing so is quite easy. A continuation you can find a very simple one with static data." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "from nornir.core.inventory import Inventory\r\n", + "\r\n", + "\r\n", + "class MyInventory(Inventory):\r\n", + " def __init__(self, **kwargs):\r\n", + " # code to get the data\r\n", + " hosts = {\r\n", + " \"host1\": {\r\n", + " \"data1\": \"value1\",\r\n", + " \"data2\": \"value2\",\r\n", + " \"data3\": \"value3\",\r\n", + " \"groups\": [\"my_group1\"],\r\n", + " },\r\n", + " \"host2\": {\r\n", + " \"data1\": \"value1\",\r\n", + " \"data2\": \"value2\",\r\n", + " \"data3\": \"value3\",\r\n", + " \"groups\": [\"my_group1\"],\r\n", + " },\r\n", + " }\r\n", + " groups = {\r\n", + " \"my_group1\": {\r\n", + " \"more_data1\": \"more_value1\",\r\n", + " \"more_data2\": \"more_value2\",\r\n", + " \"more_data3\": \"more_value3\",\r\n", + " }\r\n", + " }\r\n", + " defaults = {\"location\": \"internet\", \"language\": \"Python\"}\r\n", + "\r\n", + " # passing the data to the parent class so the data is\r\n", + " # transformed into actual Host/Group objects\r\n", + " # and set default data for all hosts\r\n", + " super().__init__(hosts, groups, defaults, **kwargs)\r\n" + ] + } + ], + "source": [ + "%cat writing_a_custom_inventory_plugin/my_inventory.py" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A dynamic inventory basically would have to retrieve the data, rearrange in a similar way as in the example above and cal ``super``. Now, let's see how to use it:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'groups': {'defaults': {'language': 'Python', 'location': 'internet'},\n", + " 'my_group1': {'more_data1': 'more_value1',\n", + " 'more_data2': 'more_value2',\n", + " 'more_data3': 'more_value3',\n", + " 'name': 'my_group1'}},\n", + " 'hosts': {'host1': {'data1': 'value1',\n", + " 'data2': 'value2',\n", + " 'data3': 'value3',\n", + " 'groups': ['my_group1'],\n", + " 'name': 'host1'},\n", + " 'host2': {'data1': 'value1',\n", + " 'data2': 'value2',\n", + " 'data3': 'value3',\n", + " 'groups': ['my_group1'],\n", + " 'name': 'host2'}}}\n" + ] + } + ], + "source": [ + "from nornir.core import InitNornir\n", + "import pprint\n", + "\n", + "nr = InitNornir(inventory=\"writing_a_custom_inventory_plugin.my_inventory.MyInventory\")\n", + "pprint.pprint(nr.inventory.to_dict())" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/howto/writing_a_custom_inventory_plugin/my_inventory.py b/docs/howto/writing_a_custom_inventory_plugin/my_inventory.py new file mode 100644 index 00000000..b0787e23 --- /dev/null +++ b/docs/howto/writing_a_custom_inventory_plugin/my_inventory.py @@ -0,0 +1,33 @@ +from nornir.core.inventory import Inventory + + +class MyInventory(Inventory): + def __init__(self, **kwargs): + # code to get the data + hosts = { + "host1": { + "data1": "value1", + "data2": "value2", + "data3": "value3", + "groups": ["my_group1"], + }, + "host2": { + "data1": "value1", + "data2": "value2", + "data3": "value3", + "groups": ["my_group1"], + }, + } + groups = { + "my_group1": { + "more_data1": "more_value1", + "more_data2": "more_value2", + "more_data3": "more_value3", + } + } + defaults = {"location": "internet", "language": "Python"} + + # passing the data to the parent class so the data is + # transformed into actual Host/Group objects + # and set default data for all hosts + super().__init__(hosts, groups, defaults, **kwargs) diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index aaba2b2c..4330e869 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -70,11 +70,11 @@ class Host(object): def __init__(self, name, groups=None, nornir=None, defaults=None, **kwargs): self.nornir = nornir self.name = name - self.groups = groups or [] + self.groups = groups if groups is not None else [] self.data = {} self.data["name"] = name self.connections = Connections() - self.defaults = defaults or {} + self.defaults = defaults if defaults is not None else {} if len(self.groups): if isinstance(groups[0], str): From b1751e6e593117ce229a02fc8f9f1f87f9a72b9f Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Thu, 16 Aug 2018 04:13:06 -0700 Subject: [PATCH 037/109] Fixing issue with napalm plugin if port but no optional_args (#231) --- nornir/plugins/connections/napalm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nornir/plugins/connections/napalm.py b/nornir/plugins/connections/napalm.py index 56090d40..d7a78ae2 100644 --- a/nornir/plugins/connections/napalm.py +++ b/nornir/plugins/connections/napalm.py @@ -35,7 +35,7 @@ def open( } parameters.update(connection_options) - if port and "port" not in connection_options["optional_args"]: + if port and "port" not in parameters["optional_args"]: parameters["optional_args"]["port"] = port network_driver = get_network_driver(platform) From eae5b96b5a39d67ffc369fb0d1b8c7983f751519 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 26 Aug 2018 10:42:58 +0200 Subject: [PATCH 038/109] use C load for ruamel.yaml (#232) --- nornir/core/configuration.py | 2 +- nornir/plugins/inventory/simple.py | 2 +- nornir/plugins/tasks/data/load_yaml.py | 2 +- tests/plugins/inventory/test_ansible.py | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/nornir/core/configuration.py b/nornir/core/configuration.py index 79bc2e87..4bee770d 100644 --- a/nornir/core/configuration.py +++ b/nornir/core/configuration.py @@ -91,7 +91,7 @@ class Config(object): def __init__(self, config_file=None, **kwargs): if config_file: with open(config_file, "r") as f: - yml = ruamel.yaml.YAML() + yml = ruamel.yaml.YAML(typ="safe") data = yml.load(f) or {} else: data = {} diff --git a/nornir/plugins/inventory/simple.py b/nornir/plugins/inventory/simple.py index bee4ea93..b77d1d43 100644 --- a/nornir/plugins/inventory/simple.py +++ b/nornir/plugins/inventory/simple.py @@ -122,7 +122,7 @@ def __init__( group_file: str = "groups.yaml", **kwargs: Any ) -> None: - yml = ruamel.yaml.YAML() + yml = ruamel.yaml.YAML(typ="safe") with open(host_file, "r") as f: hosts: HostsDict = yml.load(f) diff --git a/nornir/plugins/tasks/data/load_yaml.py b/nornir/plugins/tasks/data/load_yaml.py index 200029ea..d762210a 100644 --- a/nornir/plugins/tasks/data/load_yaml.py +++ b/nornir/plugins/tasks/data/load_yaml.py @@ -22,7 +22,7 @@ def load_yaml(task: Task, file: str): * result (``dict``): dictionary with the contents of the file """ with open(file, "r") as f: - yml = ruamel.yaml.YAML(pure=True) + yml = ruamel.yaml.YAML(typ="safe") data = yml.load(f) return Result(host=task.host, result=data) diff --git a/tests/plugins/inventory/test_ansible.py b/tests/plugins/inventory/test_ansible.py index 735a56b9..800d5bad 100644 --- a/tests/plugins/inventory/test_ansible.py +++ b/tests/plugins/inventory/test_ansible.py @@ -3,6 +3,7 @@ from nornir.plugins.inventory import ansible import pytest + import ruamel.yaml from ruamel.yaml.scanner import ScannerError @@ -11,7 +12,7 @@ def save(hosts, groups, hosts_file, groups_file): - yml = ruamel.yaml.YAML(typ="safe", pure=True) + yml = ruamel.yaml.YAML(typ="safe") yml.default_flow_style = False with open(hosts_file, "w+") as f: f.write(yml.dump(hosts)) From 2554177329c55a15f5ed6167dfa219ebbb8b9e2e Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 26 Aug 2018 11:12:38 +0200 Subject: [PATCH 039/109] fix tests --- docs/howto/transforming_inventory_data.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/howto/transforming_inventory_data.ipynb b/docs/howto/transforming_inventory_data.ipynb index b08a9e49..653b959d 100644 --- a/docs/howto/transforming_inventory_data.ipynb +++ b/docs/howto/transforming_inventory_data.ipynb @@ -65,7 +65,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'groups': {'defaults': ordereddict([('a_default_attribute', 'my_default')])},\n", + "{'groups': {'defaults': {'a_default_attribute': 'my_default'}},\n", " 'hosts': {'rtr00': {'name': 'rtr00',\n", " 'user': 'automation_user',\n", " 'username': 'automation_user'},\n", @@ -191,7 +191,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'groups': {'defaults': ordereddict([('a_default_attribute', 'my_default')])},\n", + "{'groups': {'defaults': {'a_default_attribute': 'my_default'}},\n", " 'hosts': {'rtr00': {'name': 'rtr00',\n", " 'user': 'automation_user',\n", " 'username': 'automation_user'},\n", From 9436de161f42e43c0e83af4def04d7dcde2cb8ae Mon Sep 17 00:00:00 2001 From: Dmitry Figol Date: Sun, 26 Aug 2018 20:56:00 +0200 Subject: [PATCH 040/109] Add class methods for plugins registration in Connections class This API can be used internally for built-in plugins and externally by a developer of a custom connection plugin --- nornir/core/__init__.py | 26 ++-------- nornir/core/connections.py | 67 +++++++++++++++++++++++++- nornir/core/exceptions.py | 8 +++ nornir/core/inventory.py | 2 +- nornir/plugins/connections/__init__.py | 17 +++---- tests/core/test_connections.py | 13 +++-- 6 files changed, 90 insertions(+), 43 deletions(-) diff --git a/nornir/core/__init__.py b/nornir/core/__init__.py index 8df1d343..08a112fe 100644 --- a/nornir/core/__init__.py +++ b/nornir/core/__init__.py @@ -1,12 +1,12 @@ import logging import logging.config from multiprocessing.dummy import Pool -from typing import Type from nornir.core.configuration import Config -from nornir.core.connections import ConnectionPlugin from nornir.core.task import AggregatedResult, Task -from nornir.plugins import connections +from nornir.plugins.connections import register_default_connection_plugins + +register_default_connection_plugins() class Data(object): @@ -16,12 +16,10 @@ class Data(object): Attributes: failed_hosts (list): Hosts that have failed to run a task properly - available_connections (dict): Dictionary holding available connection plugins """ def __init__(self): self.failed_hosts = set() - self.available_connections = connections.available_connections def recover_host(self, host): """Remove ``host`` from list of failed hosts.""" @@ -47,8 +45,6 @@ class Nornir(object): dry_run(``bool``): Whether if we are testing the changes or not config (:obj:`nornir.core.configuration.Config`): Configuration object config_file (``str``): Path to Yaml configuration file - available_connections (``dict``): dict of connection types that will be made available. - Defaults to :obj:`nornir.plugins.tasks.connections.available_connections` Attributes: inventory (:obj:`nornir.core.inventory.Inventory`): Inventory to work with @@ -58,14 +54,7 @@ class Nornir(object): """ def __init__( - self, - inventory, - dry_run, - config=None, - config_file=None, - available_connections=None, - logger=None, - data=None, + self, inventory, dry_run, config=None, config_file=None, logger=None, data=None ): self.logger = logger or logging.getLogger("nornir") @@ -81,9 +70,6 @@ def __init__( self.configure_logging() - if available_connections is not None: - self.data.available_connections = available_connections - def __enter__(self): return self @@ -238,10 +224,6 @@ def to_dict(self): """ Return a dictionary representing the object. """ return {"data": self.data.to_dict(), "inventory": self.inventory.to_dict()} - def get_connection_type(self, connection: str) -> Type[ConnectionPlugin]: - """Returns the class for the given connection type.""" - return self.data.available_connections[connection] - def close_connections(self, on_good=True, on_failed=False): def close_connections_task(task): task.host.close_connections() diff --git a/nornir/core/connections.py b/nornir/core/connections.py index fe50090c..3c919317 100644 --- a/nornir/core/connections.py +++ b/nornir/core/connections.py @@ -1,8 +1,12 @@ from abc import ABC, abstractmethod -from typing import Any, Dict, NoReturn, Optional +from typing import Any, Dict, NoReturn, Optional, Type from nornir.core.configuration import Config +from nornir.core.exceptions import ( + ConnectionPluginAlreadyRegistered, + ConnectionPluginNotRegistered, +) class ConnectionPlugin(ABC): @@ -53,4 +57,63 @@ def close(self) -> NoReturn: class Connections(Dict[str, ConnectionPlugin]): - pass + available: Dict[str, Type[ConnectionPlugin]] = {} + + @classmethod + def register( + cls, name: str, plugin: Type[ConnectionPlugin], force: bool = False + ) -> None: + """Registers a connection plugin with a specified name + + Args: + name: name of the connection plugin to register + plugin: defined connection plugin class + force: if set to True, will register a class with the specified name + even if a connection plugin with the same name was already + registered. Default - False + + Raises: + :obj:`nornir.core.exceptions.ConnectionPluginAlreadyRegistered` + """ + if not force and name in cls.available: + raise ConnectionPluginAlreadyRegistered( + f"Connection {name!r} was already registered" + ) + cls.available[name] = plugin + + @classmethod + def deregister(cls, name: str) -> None: + """Deregisters a registered connection plugin by its name + + Args: + name: name of the connection plugin to deregister + + Raises: + :obj:`nornir.core.exceptions.ConnectionPluginNotRegistered` + """ + if name not in cls.available: + raise ConnectionPluginNotRegistered( + f"Connection {name!r} is not registered" + ) + cls.available.pop(name) + + @classmethod + def deregister_all(cls) -> None: + """Deregisters all registered connection plugins""" + cls.available = {} + + @classmethod + def get_plugin(cls, name: str) -> Type[ConnectionPlugin]: + """Fetches the connection plugin by name if already registered + + Args: + name: name of the connection plugin + + Raises: + :obj:`nornir.core.exceptions.ConnectionPluginNotRegistered` + """ + if name not in cls.available: + raise ConnectionPluginNotRegistered( + f"Connection {name!r} is not registered" + ) + return cls.available[name] diff --git a/nornir/core/exceptions.py b/nornir/core/exceptions.py index cbceb400..e1683be9 100644 --- a/nornir/core/exceptions.py +++ b/nornir/core/exceptions.py @@ -14,6 +14,14 @@ class ConnectionNotOpen(ConnectionException): pass +class ConnectionPluginAlreadyRegistered(ConnectionException): + pass + + +class ConnectionPluginNotRegistered(ConnectionException): + pass + + class CommandError(Exception): """ Raised when there is a command error. diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 4330e869..3e2d8c57 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -329,7 +329,7 @@ def open_connection( if connection in self.connections: raise ConnectionAlreadyOpen(connection) - self.connections[connection] = self.nornir.get_connection_type(connection)() + self.connections[connection] = self.connections.get_plugin(connection)() if default_to_host_attributes: conn_params = self.get_connection_parameters(connection) self.connections[connection].open( diff --git a/nornir/plugins/connections/__init__.py b/nornir/plugins/connections/__init__.py index 5fc891fe..bdc2c574 100644 --- a/nornir/plugins/connections/__init__.py +++ b/nornir/plugins/connections/__init__.py @@ -1,16 +1,11 @@ -from typing import Dict, TYPE_CHECKING, Type - - from .napalm import Napalm from .netmiko import Netmiko from .paramiko import Paramiko - -if TYPE_CHECKING: - from nornir.core.connections import ConnectionPlugin # noqa +from nornir.core.connections import Connections -available_connections: Dict[str, Type["ConnectionPlugin"]] = { - "napalm": Napalm, - "netmiko": Netmiko, - "paramiko": Paramiko, -} +def register_default_connection_plugins() -> None: + Connections.deregister_all() + Connections.register("napalm", Napalm) + Connections.register("netmiko", Netmiko) + Connections.register("paramiko", Paramiko) diff --git a/tests/core/test_connections.py b/tests/core/test_connections.py index b400c147..25ad067f 100644 --- a/tests/core/test_connections.py +++ b/tests/core/test_connections.py @@ -1,7 +1,7 @@ from typing import Any, Dict, Optional from nornir.core.configuration import Config -from nornir.core.connections import ConnectionPlugin +from nornir.core.connections import ConnectionPlugin, Connections from nornir.core.exceptions import ConnectionAlreadyOpen, ConnectionNotOpen @@ -68,29 +68,30 @@ def validate_params(task, conn, params): class Test(object): + @classmethod + def setup_class(cls): + Connections.register("dummy", DummyConnectionPlugin) + Connections.register("dummy_no_overrides", DummyConnectionPlugin) + def test_open_and_close_connection(self, nornir): - nornir.data.available_connections["dummy"] = DummyConnectionPlugin nr = nornir.filter(name="dev2.group_1") r = nr.run(task=open_and_close_connection, num_workers=1) assert len(r) == 1 assert not r.failed def test_open_connection_twice(self, nornir): - nornir.data.available_connections["dummy"] = DummyConnectionPlugin nr = nornir.filter(name="dev2.group_1") r = nr.run(task=open_connection_twice, num_workers=1) assert len(r) == 1 assert not r.failed def test_close_not_opened_connection(self, nornir): - nornir.data.available_connections["dummy"] = DummyConnectionPlugin nr = nornir.filter(name="dev2.group_1") r = nr.run(task=close_not_opened_connection, num_workers=1) assert len(r) == 1 assert not r.failed def test_context_manager(self, nornir): - nornir.data.available_connections["dummy"] = DummyConnectionPlugin with nornir.filter(name="dev2.group_1") as nr: nr.run(task=a_task) assert "dummy" in nr.inventory.hosts["dev2.group_1"].connections @@ -98,7 +99,6 @@ def test_context_manager(self, nornir): nornir.data.reset_failed_hosts() def test_validate_params_simple(self, nornir): - nornir.data.available_connections["dummy_no_overrides"] = DummyConnectionPlugin params = { "hostname": "127.0.0.1", "username": "root", @@ -118,7 +118,6 @@ def test_validate_params_simple(self, nornir): assert not r.failed def test_validate_params_overrides(self, nornir): - nornir.data.available_connections["dummy"] = DummyConnectionPlugin params = { "hostname": "overriden_hostname", "username": "root", From c135c196717f5ec78b128910aa2b76b2a2cb4a8b Mon Sep 17 00:00:00 2001 From: Dmitry Figol Date: Mon, 27 Aug 2018 18:34:10 +0200 Subject: [PATCH 041/109] Add tests for connections plugin registration/deregistration --- tests/core/test_connections.py | 61 +++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/tests/core/test_connections.py b/tests/core/test_connections.py index 25ad067f..1bdfc849 100644 --- a/tests/core/test_connections.py +++ b/tests/core/test_connections.py @@ -1,8 +1,16 @@ from typing import Any, Dict, Optional +import pytest + from nornir.core.configuration import Config from nornir.core.connections import ConnectionPlugin, Connections -from nornir.core.exceptions import ConnectionAlreadyOpen, ConnectionNotOpen +from nornir.core.exceptions import ( + ConnectionAlreadyOpen, + ConnectionNotOpen, + ConnectionPluginNotRegistered, + ConnectionPluginAlreadyRegistered, +) +from nornir.plugins.connections import register_default_connection_plugins class DummyConnectionPlugin(ConnectionPlugin): @@ -30,6 +38,10 @@ def close(self) -> None: self.connection = False +class AnotherDummyConnectionPlugin(DummyConnectionPlugin): + pass + + def open_and_close_connection(task): task.host.open_connection("dummy") assert "dummy" in task.host.connections @@ -130,3 +142,50 @@ def test_validate_params_overrides(self, nornir): r = nr.run(task=validate_params, conn="dummy", params=params, num_workers=1) assert len(r) == 1 assert not r.failed + + +class TestConnectionPluginsRegistration(object): + def setup_method(self, method): + Connections.deregister_all() + Connections.register("dummy", DummyConnectionPlugin) + Connections.register("another_dummy", AnotherDummyConnectionPlugin) + + def teardown_method(self, method): + register_default_connection_plugins() + + def test_count(self): + assert len(Connections.available) == 2 + + def test_register_new(self): + Connections.register("new_dummy", DummyConnectionPlugin) + assert "new_dummy" in Connections.available + + def test_register_existing(self): + with pytest.raises(ConnectionPluginAlreadyRegistered): + Connections.register("dummy", DummyConnectionPlugin) + + def test_register_existing_force(self): + Connections.register("dummy", AnotherDummyConnectionPlugin, force=True) + assert Connections.available["dummy"] == AnotherDummyConnectionPlugin + + def test_deregister_existing(self): + Connections.deregister("dummy") + assert len(Connections.available) == 1 + assert "dummy" not in Connections.available + + def test_deregister_nonexistent(self): + with pytest.raises(ConnectionPluginNotRegistered): + Connections.deregister("nonexistent_dummy") + + def test_deregister_all(self): + Connections.deregister_all() + assert Connections.available == {} + + def test_get_plugin(self): + assert Connections.get_plugin("dummy") == DummyConnectionPlugin + assert Connections.get_plugin("another_dummy") == AnotherDummyConnectionPlugin + assert len(Connections.available) == 2 + + def test_nonexistent_plugin(self): + with pytest.raises(ConnectionPluginNotRegistered): + Connections.get_plugin("nonexistent_dummy") From 9d153d4682cbed075c63b0d647d78a2c21faf99f Mon Sep 17 00:00:00 2001 From: Dmitry Figol Date: Tue, 28 Aug 2018 14:56:01 +0200 Subject: [PATCH 042/109] Modify connection plugins registration * Remove deregister_all() from register_default_connection_plugins() * Remove force argument from Connections.register() * Do not raise an exception if the same plugin with the same name is registered again * Update tests --- nornir/core/connections.py | 20 ++++++++++---------- nornir/plugins/connections/__init__.py | 1 - tests/core/test_connections.py | 14 ++++++++------ 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/nornir/core/connections.py b/nornir/core/connections.py index 3c919317..5e26397e 100644 --- a/nornir/core/connections.py +++ b/nornir/core/connections.py @@ -60,26 +60,26 @@ class Connections(Dict[str, ConnectionPlugin]): available: Dict[str, Type[ConnectionPlugin]] = {} @classmethod - def register( - cls, name: str, plugin: Type[ConnectionPlugin], force: bool = False - ) -> None: + def register(cls, name: str, plugin: Type[ConnectionPlugin]) -> None: """Registers a connection plugin with a specified name Args: name: name of the connection plugin to register plugin: defined connection plugin class - force: if set to True, will register a class with the specified name - even if a connection plugin with the same name was already - registered. Default - False Raises: - :obj:`nornir.core.exceptions.ConnectionPluginAlreadyRegistered` + :obj:`nornir.core.exceptions.ConnectionPluginAlreadyRegistered` if + another plugin with the specified name was already registered """ - if not force and name in cls.available: + existing_plugin = cls.available.get(name) + if existing_plugin is None: + cls.available[name] = plugin + elif existing_plugin != plugin: raise ConnectionPluginAlreadyRegistered( - f"Connection {name!r} was already registered" + f"Connection plugin {plugin.__name__} can't be registered as " + f"{name!r} because plugin {existing_plugin.__name__} " + f"was already registered under this name" ) - cls.available[name] = plugin @classmethod def deregister(cls, name: str) -> None: diff --git a/nornir/plugins/connections/__init__.py b/nornir/plugins/connections/__init__.py index bdc2c574..fb149889 100644 --- a/nornir/plugins/connections/__init__.py +++ b/nornir/plugins/connections/__init__.py @@ -5,7 +5,6 @@ def register_default_connection_plugins() -> None: - Connections.deregister_all() Connections.register("napalm", Napalm) Connections.register("netmiko", Netmiko) Connections.register("paramiko", Paramiko) diff --git a/tests/core/test_connections.py b/tests/core/test_connections.py index 1bdfc849..54c277d4 100644 --- a/tests/core/test_connections.py +++ b/tests/core/test_connections.py @@ -82,6 +82,7 @@ def validate_params(task, conn, params): class Test(object): @classmethod def setup_class(cls): + Connections.deregister_all() Connections.register("dummy", DummyConnectionPlugin) Connections.register("dummy_no_overrides", DummyConnectionPlugin) @@ -151,6 +152,7 @@ def setup_method(self, method): Connections.register("another_dummy", AnotherDummyConnectionPlugin) def teardown_method(self, method): + Connections.deregister_all() register_default_connection_plugins() def test_count(self): @@ -160,13 +162,13 @@ def test_register_new(self): Connections.register("new_dummy", DummyConnectionPlugin) assert "new_dummy" in Connections.available - def test_register_existing(self): - with pytest.raises(ConnectionPluginAlreadyRegistered): - Connections.register("dummy", DummyConnectionPlugin) + def test_register_already_registered_same(self): + Connections.register("dummy", DummyConnectionPlugin) + assert Connections.available["dummy"] == DummyConnectionPlugin - def test_register_existing_force(self): - Connections.register("dummy", AnotherDummyConnectionPlugin, force=True) - assert Connections.available["dummy"] == AnotherDummyConnectionPlugin + def test_register_already_registered_new(self): + with pytest.raises(ConnectionPluginAlreadyRegistered): + Connections.register("dummy", AnotherDummyConnectionPlugin) def test_deregister_existing(self): Connections.deregister("dummy") From e39e614b2650522e43d3538cbc969e46e1157145 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Thu, 30 Aug 2018 10:51:57 +0200 Subject: [PATCH 043/109] refactor configuration --- nornir/core/__init__.py | 62 +---- nornir/core/configuration.py | 293 ++++++++++------------ nornir/plugins/connections/paramiko.py | 2 +- requirements.txt | 1 + setup.cfg | 11 + tests/core/test_InitNornir.py | 32 ++- tests/core/test_InitNornir/a_config.yaml | 9 +- tests/core/test_configuration.py | 78 ++---- tests/core/test_configuration/config.yaml | 7 +- 9 files changed, 205 insertions(+), 290 deletions(-) diff --git a/nornir/core/__init__.py b/nornir/core/__init__.py index 8df1d343..599d63a5 100644 --- a/nornir/core/__init__.py +++ b/nornir/core/__init__.py @@ -79,8 +79,6 @@ def __init__( else: self.config = config or Config() - self.configure_logging() - if available_connections is not None: self.data.available_connections = available_connections @@ -94,53 +92,6 @@ def __exit__(self, exc_type, exc_val, exc_tb): def dry_run(self): return self.data.dry_run - def configure_logging(self): - dictConfig = self.config.logging_dictConfig or { - "version": 1, - "disable_existing_loggers": False, - "formatters": {"simple": {"format": self.config.logging_format}}, - "handlers": {}, - "loggers": {}, - "root": { - "level": "CRITICAL" - if self.config.logging_loggers - else self.config.logging_level.upper(), # noqa - "handlers": [], - "formatter": "simple", - }, - } - handlers_list = [] - if self.config.logging_file: - dictConfig["root"]["handlers"].append("info_file_handler") - handlers_list.append("info_file_handler") - dictConfig["handlers"]["info_file_handler"] = { - "class": "logging.handlers.RotatingFileHandler", - "level": "NOTSET", - "formatter": "simple", - "filename": self.config.logging_file, - "maxBytes": 10485760, - "backupCount": 20, - "encoding": "utf8", - } - if self.config.logging_to_console: - dictConfig["root"]["handlers"].append("info_console") - handlers_list.append("info_console") - dictConfig["handlers"]["info_console"] = { - "class": "logging.StreamHandler", - "level": "NOTSET", - "formatter": "simple", - "stream": "ext://sys.stdout", - } - - for logger in self.config.logging_loggers: - dictConfig["loggers"][logger] = { - "level": self.config.logging_level.upper(), - "handlers": handlers_list, - } - - if dictConfig["root"]["handlers"]: - logging.config.dictConfig(dictConfig) - def filter(self, *args, **kwargs): """ See :py:meth:`nornir.core.inventory.Inventory.filter` @@ -249,7 +200,7 @@ def close_connections_task(task): self.run(task=close_connections_task, on_good=on_good, on_failed=on_failed) -def InitNornir(config_file="", dry_run=False, **kwargs): +def InitNornir(config_file="", dry_run=False, configure_logging=True, **kwargs): """ Arguments: config_file(str): Path to the configuration file (optional) @@ -260,11 +211,12 @@ def InitNornir(config_file="", dry_run=False, **kwargs): Returns: :obj:`nornir.core.Nornir`: fully instantiated and configured """ - conf = Config(config_file=config_file, **kwargs) + conf = Config(path=config_file, **kwargs) + if configure_logging: + conf.logging.configure() - inv_class = conf.inventory - inv_args = getattr(conf, inv_class.__name__, {}) - transform_function = conf.transform_function - inv = inv_class(transform_function=transform_function, **inv_args) + inv_class = conf.inventory.get_plugin() + transform_function = conf.inventory.get_transform_function() + inv = inv_class(transform_function=transform_function, **conf.inventory.options) return Nornir(inventory=inv, dry_run=dry_run, config=conf) diff --git a/nornir/core/configuration.py b/nornir/core/configuration.py index 4bee770d..d08082b8 100644 --- a/nornir/core/configuration.py +++ b/nornir/core/configuration.py @@ -1,179 +1,148 @@ import importlib -import os +import logging +import logging.config +from typing import Any, Callable, Dict, List, Optional +from pydantic import BaseSettings import ruamel.yaml -CONF = { - "inventory": { - "description": "Path to inventory modules.", - "type": "str", - "default": "nornir.plugins.inventory.simple.SimpleInventory", - }, - "transform_function": { - "description": "Path to transform function. The transform_function you provide " - "will run against each host in the inventory.", - "type": "str", - "default": {}, - }, - "jinja_filters": { - "description": "Path to callable returning jinja filters to be used.", - "type": "str", - "default": {}, - }, - "num_workers": { - "description": "Number of Nornir worker processes that are run at the same time, " - "configuration can be overridden on individual tasks by using the " - "`num_workers` argument to (:obj:`nornir.core.Nornir.run`)", - "type": "int", - "default": 20, - }, - "raise_on_error": { - "description": "If set to ``True``, (:obj:`nornir.core.Nornir.run`) method of will raise " - "an exception if at least a host failed.", - "type": "bool", - "default": False, - }, - "ssh_config_file": { - "description": "User ssh_config_file", - "type": "str", - "default": os.path.join(os.path.expanduser("~"), ".ssh", "config"), - "default_doc": "~/.ssh/config", - }, - "logging_dictConfig": { - "description": "Configuration dictionary schema supported by the logging subsystem. " - "Overrides rest of logging_* parameters.", - "type": "dict", - "default": {}, - }, - "logging_level": { - "description": "Logging level. Can be any supported level by the logging subsystem", - "type": "str", - "default": "debug", - }, - "logging_file": { - "description": "Logging file. Empty string disables logging to file.", - "type": "str", - "default": "nornir.log", - }, - "logging_format": { - "description": "Logging format", - "type": "str", - "default": "%(asctime)s - %(name)12s - %(levelname)8s - %(funcName)10s() - %(message)s", - }, - "logging_to_console": { - "description": "Whether to log to stdout or not.", - "type": "bool", - "default": False, - }, - "logging_loggers": { - "description": "List of loggers to configure. This allows you to enable logging for " - "multiple loggers. For instance, you could enable logging for both nornir " - "and paramiko or just for paramiko. An empty list will enable logging for " - "all loggers.", - "type": "list", - "default": ["nornir"], - }, -} - -types = {"int": int, "str": str} - - -class Config(object): +class SSHConfig(BaseSettings): """ - This object handles the configuration of Nornir. - - Arguments: - config_file(``str``): Yaml configuration file. + Args: + config_file: User ssh_config_file """ - def __init__(self, config_file=None, **kwargs): - if config_file: - with open(config_file, "r") as f: - yml = ruamel.yaml.YAML(typ="safe") - data = yml.load(f) or {} - else: - data = {} + config_file: str = "~/.ssh/config" - for parameter, param_conf in CONF.items(): - self._assign_property(parameter, param_conf, data) + class Config: + env_prefix = "NORNIR_SSH_" + + +class Inventory(BaseSettings): + """ + Args: + plugin: Path to inventory modules. + transform_function: Path to transform function. The transform_function you provide + will run against each host in the inventory + options: Arguments to pass to the inventory plugin + """ - for k, v in data.items(): - if k not in CONF: - setattr(self, k, v) + plugin: Any = "nornir.plugins.inventory.simple.SimpleInventory" + options: Dict[str, Any] = {} + transform_function: Any = "" + + def get_plugin(self) -> Optional[Callable[..., Any]]: + return _resolve_import_from_string(self.plugin) + + def get_transform_function(self) -> Optional[Callable[..., Any]]: + return _resolve_import_from_string(self.transform_function) + + class Config: + env_prefix = "NORNIR_INVENTORY_" + + +class Logging(BaseSettings): + level: str = "debug" + file: str = "nornir.log" + format: str = "%(asctime)s - %(name)12s - %(levelname)8s - %(funcName)10s() - %(message)s" + to_console: bool = False + loggers: List[str] = ["nornir"] + + class Config: + env_prefix = "NORNIR_LOGGING_" + + def configure(self): + dictConfig = { + "version": 1, + "disable_existing_loggers": False, + "formatters": {"simple": {"format": self.format}}, + "handlers": {}, + "loggers": {}, + "root": { + "level": "CRITICAL" if self.loggers else self.level.upper(), + "handlers": [], + "formatter": "simple", + }, + } + handlers_list = [] + if self.file: + dictConfig["root"]["handlers"].append("info_file_handler") + handlers_list.append("info_file_handler") + dictConfig["handlers"]["info_file_handler"] = { + "class": "logging.handlers.RotatingFileHandler", + "level": "NOTSET", + "formatter": "simple", + "filename": self.file, + "maxBytes": 10485760, + "backupCount": 20, + "encoding": "utf8", + } + if self.to_console: + dictConfig["root"]["handlers"].append("info_console") + handlers_list.append("info_console") + dictConfig["handlers"]["info_console"] = { + "class": "logging.StreamHandler", + "level": "NOTSET", + "formatter": "simple", + "stream": "ext://sys.stdout", + } + + for logger in self.loggers: + dictConfig["loggers"][logger] = { + "level": self.level.upper(), + "handlers": handlers_list, + } + + if dictConfig["root"]["handlers"]: + logging.config.dictConfig(dictConfig) + + +class Config(BaseSettings): + """ + Args: + inventory: Dictionary with Inventory options + jinja_filters: Path to callable returning jinja filters to be used + raise_on_error: If set to ``True``, (:obj:`nornir.core.Nornir.run`) method of + will raise an exception if at least a host failed + num_workers: Number of Nornir worker processes that are run at the same time + configuration can be overridden on individual tasks by using the - for k, v in kwargs.items(): - setattr(self, k, v) - resolve_imports = ["inventory", "transform_function", "jinja_filters"] - for r in resolve_imports: - obj = self._resolve_import_from_string(kwargs.get(r, getattr(self, r))) - setattr(self, r, obj) + """ - callable_func = ["jinja_filters"] - for c in callable_func: - func = getattr(self, c) - if func: - setattr(self, c, func()) + inventory: Inventory = Inventory() + jinja_filters: str = "" + num_workers: int = 20 + raise_on_error: bool = False + ssh: SSHConfig = SSHConfig() + user_defined: Dict[str, Any] = {} + logging: Logging = Logging() - def string_to_bool(self, v): - if v.lower() in ["false", "no", "n", "off", "0"]: - return False + class Config: + env_prefix = "NORNIR_" + def __init__(self, path: str = "", **kwargs) -> None: + if path: + with open(path, "r") as f: + yml = ruamel.yaml.YAML(typ="safe") + data = yml.load(f) or {} + data.update(kwargs) else: - return True - - def _assign_property(self, parameter, param_conf, data): - v = None - if param_conf["type"] in ("bool", "int", "str"): - env = param_conf.get("env") or "BRIGADE_" + parameter.upper() - v = os.environ.get(env) - if v is None: - v = data.get(parameter, param_conf["default"]) - else: - if param_conf["type"] == "bool": - v = self.string_to_bool(v) - else: - v = types[param_conf["type"]](v) - setattr(self, parameter, v) - - def get(self, parameter, env=None, default=None, parameter_type="str", root=""): - """ - Retrieve a custom parameter from the configuration. - - Arguments: - parameter(str): Name of the parameter to retrieve - env(str): Environment variable name to retrieve the object from - default: default value in case no parameter is found - parameter_type(str): if a value is found cast the variable to this type - root(str): parent key in the configuration file where to look for the parameter - """ - value = os.environ.get(env) if env else None - if value is None: - if root: - d = getattr(self, root, {}) - value = d.get(parameter, default) - else: - value = getattr(self, parameter, default) - if parameter_type in [bool, "bool"]: - if not isinstance(value, bool): - value = self.string_to_bool(value) - else: - value = types[str(parameter_type)](value) - return value - - def _resolve_import_from_string(self, import_path): - """ - Resolves import from a string. Checks if callable or path is given. - - Arguments: - import_path(str): path of the import - """ - if not import_path or callable(import_path): - return import_path - - module_name = ".".join(import_path.split(".")[:-1]) - obj_name = import_path.split(".")[-1] - module = importlib.import_module(module_name) - return getattr(module, obj_name) + data = kwargs + data["ssh"] = SSHConfig(**data.pop("ssh", {})) + data["inventory"] = Inventory(**data.pop("inventory", {})) + data["logging"] = Logging(**data.pop("logging", {})) + super().__init__(**data) + + +def _resolve_import_from_string(import_path: Any) -> Optional[Callable[..., Any]]: + if not import_path: + return None + elif callable(import_path): + return import_path + module_name = ".".join(import_path.split(".")[:-1]) + obj_name = import_path.split(".")[-1] + module = importlib.import_module(module_name) + return getattr(module, obj_name) diff --git a/nornir/plugins/connections/paramiko.py b/nornir/plugins/connections/paramiko.py index abb1f49a..aa9ade1b 100644 --- a/nornir/plugins/connections/paramiko.py +++ b/nornir/plugins/connections/paramiko.py @@ -33,7 +33,7 @@ def open( client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh_config = paramiko.SSHConfig() - ssh_config_file = configuration.ssh_config_file # type: ignore + ssh_config_file = configuration.ssh.config_file # type: ignore if os.path.exists(ssh_config_file): with open(ssh_config_file) as f: ssh_config.parse(f) diff --git a/requirements.txt b/requirements.txt index bf1ff10b..4ca5aedd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ future requests ruamel.yaml mypy_extensions +pydantic diff --git a/setup.cfg b/setup.cfg index e9a03184..ea33b94a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,15 @@ warn_redundant_casts = True warn_unused_configs = True ignore_missing_imports = True +[mypy-nornir.core.configuration] +check_untyped_defs = True +disallow_any_generics = True +# Turn on the next flag once the whole codebase is annotated (Phase 2) +disallow_untyped_calls = True +strict_optional = True +warn_unused_ignores = True +ignore_errors = False + [mypy-nornir.core.connections] check_untyped_defs = True disallow_any_generics = True @@ -49,3 +58,5 @@ ignore_errors = False [mypy-nornir.*] ignore_errors = True +[mypy-tests.*] +ignore_errors = True diff --git a/tests/core/test_InitNornir.py b/tests/core/test_InitNornir.py index 5a6273cb..195a2dae 100644 --- a/tests/core/test_InitNornir.py +++ b/tests/core/test_InitNornir.py @@ -41,10 +41,12 @@ def test_InitNornir_file(self): def test_InitNornir_programmatically(self): nr = InitNornir( num_workers=100, - inventory="nornir.plugins.inventory.simple.SimpleInventory", - SimpleInventory={ - "host_file": "tests/inventory_data/hosts.yaml", - "group_file": "tests/inventory_data/groups.yaml", + inventory={ + "plugin": "nornir.plugins.inventory.simple.SimpleInventory", + "options": { + "host_file": "tests/inventory_data/hosts.yaml", + "group_file": "tests/inventory_data/groups.yaml", + }, }, ) assert not nr.dry_run @@ -64,21 +66,28 @@ def test_InitNornir_combined(self): def test_InitNornir_different_inventory_by_string(self): nr = InitNornir( config_file=os.path.join(dir_path, "a_config.yaml"), - inventory="tests.core.test_InitNornir.StringInventory", + inventory={"plugin": "tests.core.test_InitNornir.StringInventory"}, ) assert isinstance(nr.inventory, StringInventory) def test_InitNornir_different_inventory_imported(self): nr = InitNornir( config_file=os.path.join(dir_path, "a_config.yaml"), - inventory=StringInventory, + inventory={"plugin": StringInventory}, ) assert isinstance(nr.inventory, StringInventory) def test_InitNornir_different_transform_function_by_string(self): nr = InitNornir( config_file=os.path.join(dir_path, "a_config.yaml"), - transform_function="tests.core.test_InitNornir.transform_func", + inventory={ + "plugin": "nornir.plugins.inventory.simple.SimpleInventory", + "transform_function": "tests.core.test_InitNornir.transform_func", + "options": { + "host_file": "tests/inventory_data/hosts.yaml", + "group_file": "tests/inventory_data/groups.yaml", + }, + }, ) for value in nr.inventory.hosts.values(): assert value.processed_by_transform_function @@ -86,7 +95,14 @@ def test_InitNornir_different_transform_function_by_string(self): def test_InitNornir_different_transform_function_imported(self): nr = InitNornir( config_file=os.path.join(dir_path, "a_config.yaml"), - transform_function=transform_func, + inventory={ + "plugin": "nornir.plugins.inventory.simple.SimpleInventory", + "transform_function": transform_func, + "options": { + "host_file": "tests/inventory_data/hosts.yaml", + "group_file": "tests/inventory_data/groups.yaml", + }, + }, ) for value in nr.inventory.hosts.values(): assert value.processed_by_transform_function diff --git a/tests/core/test_InitNornir/a_config.yaml b/tests/core/test_InitNornir/a_config.yaml index a0fddd4f..abc15dd3 100644 --- a/tests/core/test_InitNornir/a_config.yaml +++ b/tests/core/test_InitNornir/a_config.yaml @@ -1,6 +1,7 @@ --- num_workers: 100 -inventory: nornir.plugins.inventory.simple.SimpleInventory -SimpleInventory: - host_file: "tests/inventory_data/hosts.yaml" - group_file: "tests/inventory_data/groups.yaml" +inventory: + plugin: nornir.plugins.inventory.simple.SimpleInventory + options: + host_file: "tests/inventory_data/hosts.yaml" + group_file: "tests/inventory_data/groups.yaml" diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py index 17002fc3..df59501a 100644 --- a/tests/core/test_configuration.py +++ b/tests/core/test_configuration.py @@ -1,6 +1,7 @@ import os from nornir.core.configuration import Config +from nornir.plugins.inventory.simple import SimpleInventory # import pytest @@ -13,84 +14,47 @@ class Test(object): def test_configuration_empty(self): config = Config( - config_file=os.path.join(dir_path, "empty.yaml"), - arg1=1, - arg2=False, - arg3=None, - arg4="asd", + os.path.join(dir_path, "empty.yaml"), user_defined={"asd": "qwe"} ) + assert config.user_defined["asd"] == "qwe" assert config.num_workers == 20 assert not config.raise_on_error - assert config.arg1 == 1 - assert config.arg2 is False - assert config.arg3 is None - assert config.arg4 == "asd" + assert ( + config.inventory.plugin == "nornir.plugins.inventory.simple.SimpleInventory" + ) + assert config.inventory.get_plugin() == SimpleInventory def test_configuration_normal(self): - config = Config( - config_file=os.path.join(dir_path, "config.yaml"), - arg1=1, - arg2=False, - arg3=None, - arg4="asd", - ) + config = Config(os.path.join(dir_path, "config.yaml")) assert config.num_workers == 10 assert not config.raise_on_error - assert config.arg1 == 1 - assert config.arg2 is False - assert config.arg3 is None - assert config.arg4 == "asd" + assert config.inventory.plugin == "something" def test_configuration_normal_override_argument(self): config = Config( - config_file=os.path.join(dir_path, "config.yaml"), - num_workers=20, - raise_on_error=True, + os.path.join(dir_path, "config.yaml"), num_workers=20, raise_on_error=True ) assert config.num_workers == 20 assert config.raise_on_error def test_configuration_normal_override_env(self): - os.environ["BRIGADE_NUM_WORKERS"] = "30" - os.environ["BRIGADE_RAISE_ON_ERROR"] = "1" - config = Config(config_file=os.path.join(dir_path, "config.yaml")) + os.environ["NORNIR_NUM_WORKERS"] = "30" + os.environ["NORNIR_RAISE_ON_ERROR"] = "1" + os.environ["NORNIR_SSH_CONFIG_FILE"] = "/user/ssh_config" + config = Config() assert config.num_workers == 30 assert config.raise_on_error - os.environ.pop("BRIGADE_NUM_WORKERS") - os.environ.pop("BRIGADE_RAISE_ON_ERROR") + assert config.ssh.config_file == "/user/ssh_config" + os.environ.pop("NORNIR_NUM_WORKERS") + os.environ.pop("NORNIR_RAISE_ON_ERROR") + os.environ.pop("NORNIR_SSH_CONFIG_FILE") def test_configuration_bool_env(self): - os.environ["BRIGADE_RAISE_ON_ERROR"] = "0" + os.environ["NORNIR_RAISE_ON_ERROR"] = "0" config = Config() assert config.num_workers == 20 assert not config.raise_on_error def test_get_user_defined_from_file(self): - config = Config(config_file=os.path.join(dir_path, "config.yaml")) - assert ( - config.get("user_defined", env="USER_DEFINED", default="qweqwe") == "asdasd" - ) - - def test_get_user_defined_from_default(self): - config = Config() - assert ( - config.get("user_defined", env="USER_DEFINED", default="qweqwe") == "qweqwe" - ) - - def test_get_user_defined_from_env(self): - os.environ["USER_DEFINED"] = "zxczxc" - config = Config(config_file=os.path.join(dir_path, "config.yaml")) - assert ( - config.get("user_defined", env="USER_DEFINED", default="qweqwe") == "zxczxc" - ) - os.environ.pop("USER_DEFINED") - - def test_get_user_defined_from_env_bool(self): - os.environ["USER_DEFINED"] = "0" - config = Config() - assert not config.get("user_defined", env="USER_DEFINED", parameter_type="bool") - os.environ.pop("USER_DEFINED") - - def test_get_user_defined_nested(self): - config = Config(config_file=os.path.join(dir_path, "config.yaml")) - assert config.get("user_defined", root="my_root") == "i am nested" + config = Config(os.path.join(dir_path, "config.yaml")) + assert config.user_defined["asd"] == "qwe" diff --git a/tests/core/test_configuration/config.yaml b/tests/core/test_configuration/config.yaml index e66cb710..bf88d73b 100644 --- a/tests/core/test_configuration/config.yaml +++ b/tests/core/test_configuration/config.yaml @@ -1,6 +1,7 @@ --- num_workers: 10 raise_on_error: false -user_defined: "asdasd" -my_root: - user_defined: "i am nested" +inventory: + plugin: something +user_defined: + asd: qwe From f9dfb4099a5a3df91fd078c45b9958862b1179eb Mon Sep 17 00:00:00 2001 From: David Barroso Date: Thu, 30 Aug 2018 11:00:21 +0200 Subject: [PATCH 044/109] trying to make mypy happy --- nornir/core/configuration.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/nornir/core/configuration.py b/nornir/core/configuration.py index d08082b8..e08128b3 100644 --- a/nornir/core/configuration.py +++ b/nornir/core/configuration.py @@ -54,23 +54,27 @@ class Config: env_prefix = "NORNIR_LOGGING_" def configure(self): + rootHandlers: List[Any] = [] + root = { + "level": "CRITICAL" if self.loggers else self.level.upper(), + "handlers": rootHandlers, + "formatter": "simple", + } + handlers: Dict[str, Any] = {} + loggers: Dict[str, Any] = {} dictConfig = { "version": 1, "disable_existing_loggers": False, "formatters": {"simple": {"format": self.format}}, - "handlers": {}, - "loggers": {}, - "root": { - "level": "CRITICAL" if self.loggers else self.level.upper(), - "handlers": [], - "formatter": "simple", - }, + "handlers": handlers, + "loggers": loggers, + "root": root, } handlers_list = [] if self.file: - dictConfig["root"]["handlers"].append("info_file_handler") + root["handlers"].append("info_file_handler") handlers_list.append("info_file_handler") - dictConfig["handlers"]["info_file_handler"] = { + handlers["info_file_handler"] = { "class": "logging.handlers.RotatingFileHandler", "level": "NOTSET", "formatter": "simple", @@ -80,9 +84,9 @@ def configure(self): "encoding": "utf8", } if self.to_console: - dictConfig["root"]["handlers"].append("info_console") + root["handlers"].append("info_console") handlers_list.append("info_console") - dictConfig["handlers"]["info_console"] = { + handlers["info_console"] = { "class": "logging.StreamHandler", "level": "NOTSET", "formatter": "simple", @@ -90,12 +94,9 @@ def configure(self): } for logger in self.loggers: - dictConfig["loggers"][logger] = { - "level": self.level.upper(), - "handlers": handlers_list, - } + loggers[logger] = {"level": self.level.upper(), "handlers": handlers_list} - if dictConfig["root"]["handlers"]: + if rootHandlers: logging.config.dictConfig(dictConfig) From 5d6c31d4e34d96e6825d7fedbc6e7aa4ae8e6600 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Fri, 31 Aug 2018 18:13:10 +0200 Subject: [PATCH 045/109] progress --- nornir/core/inventory.py | 314 ++++-------------- nornir/core/old_inventory.py | 511 +++++++++++++++++++++++++++++ nornir/core/serializer.py | 95 ++++++ nornir/plugins/inventory/simple.py | 146 +-------- tests/conftest.py | 110 +++---- tests/core/test_inventory.py | 320 ++++++------------ tests/inventory_data/defaults.yaml | 8 + tests/inventory_data/groups.yaml | 41 ++- tests/inventory_data/hosts.yaml | 80 ++--- 9 files changed, 915 insertions(+), 710 deletions(-) create mode 100644 nornir/core/old_inventory.py create mode 100644 nornir/core/serializer.py create mode 100644 tests/inventory_data/defaults.yaml diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 3e2d8c57..d3116484 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -1,89 +1,55 @@ -import getpass -from collections import Mapping -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from nornir.core.configuration import Config from nornir.core.connections import Connections from nornir.core.exceptions import ConnectionAlreadyOpen, ConnectionNotOpen +GroupsDict = None # DELETEME +HostsDict = None # DELETEME +VarsDict = None # DELETEME -VarsDict = Dict[str, Any] -HostsDict = Dict[str, VarsDict] -GroupsDict = Dict[str, VarsDict] - - -class Host(object): - """ - Represents a host. - - Arguments: - name (str): Name of the host - group (:obj:`Group`, optional): Group the host belongs to - nornir (:obj:`nornir.core.Nornir`): Reference to the parent nornir object - **kwargs: Host data - - Attributes: - name (str): Name of the host - groups (list of :obj:`Group`): Groups the host belongs to - defaults (``dict``): Default values for host/group data - data (dict): data about the device - connections (``dict``): Already established connections - - Note: - - You can access the host data in two ways: - - 1. Via the ``data`` attribute - In this case you will get access - **only** to the data that belongs to the host. - 2. Via the host itself as a dict - :obj:`Host` behaves like a - dict. The difference between accessing data via the ``data`` attribute - and directly via the host itself is that the latter will also - return the data if it's available via a parent :obj:`Group`. - - For instance:: - - --- - # hosts - my_host: - ip: 1.2.3.4 - groups: [bma] - - --- - # groups - bma: - site: bma - - defaults: - domain: acme.com - - * ``my_host.data["ip"]`` will return ``1.2.3.4`` - * ``my_host["ip"]`` will return ``1.2.3.4`` - * ``my_host.data["site"]`` will ``fail`` - * ``my_host["site"]`` will return ``bma`` - * ``my_host.data["domain"]`` will ``fail`` - * ``my_host.group.data["domain"]`` will ``fail`` - * ``my_host["domain"]`` will return ``acme.com`` - * ``my_host.group["domain"]`` will return ``acme.com`` - * ``my_host.group.group.data["domain"]`` will return ``acme.com`` - """ - - def __init__(self, name, groups=None, nornir=None, defaults=None, **kwargs): - self.nornir = nornir - self.name = name - self.groups = groups if groups is not None else [] - self.data = {} - self.data["name"] = name + +class ElementData(object): + __slots__ = ( + "hostname", + "port", + "username", + "password", + "platform", + "groups", + "data", + "connections", + ) + + def __init__( + self, + hostname: Optional[str] = None, + port: Optional[str] = None, + username: Optional[str] = None, + password: Optional[str] = None, + platform: Optional[str] = None, + groups: Optional[List["Group"]] = None, + data: Optional[Dict[str, Any]] = None, + ) -> None: + self.hostname = hostname + self.port = port + self.username = username + self.password = password + self.platform = platform + self.groups = groups or [] + self.data = data or {} self.connections = Connections() - self.defaults = defaults if defaults is not None else {} - if len(self.groups): - if isinstance(groups[0], str): - self.data["groups"] = groups - else: - self.data["groups"] = [g.name for g in groups] - for k, v in kwargs.items(): - self.data[k] = v +class Host(ElementData): + __slots__ = ("name", "defaults") + + def __init__( + self, name: str, defaults: Optional[ElementData] = None, *args, **kwargs + ) -> None: + self.name = name + self.defaults = defaults or ElementData() + super().__init__(*args, **kwargs) def _resolve_data(self): processed = [] @@ -96,7 +62,7 @@ def _resolve_data(self): if k not in processed: processed.append(k) result[k] = v - for k, v in self.defaults.items(): + for k, v in self.defaults.data.items(): if k not in processed: processed.append(k) result[k] = v @@ -149,12 +115,26 @@ def __getitem__(self, item): if r: return r - r = self.defaults.get(item) + r = self.defaults.data.get(item) if r: return r raise + def __getattribute__(self, name): + if name not in ("hostname", "port", "username", "password", "platform"): + return object.__getattribute__(self, name) + v = object.__getattribute__(self, name) + if v is None: + for g in self.groups: + r = object.__getattribute__(g, name) + if r is not None: + return r + + return getattr(self.defaults, name) + else: + return v + def __setitem__(self, item, value): self.data[item] = value @@ -168,7 +148,7 @@ def __str__(self): return self.name def __repr__(self): - return "{}: {}".format(self.__class__.__name__, self.name) + return "{}: {}".format(self.__class__.__name__, self.hostname or "") def get(self, item, default=None): """ @@ -196,51 +176,6 @@ def nornir(self, value): if not getattr(self, "_nornir", None): self._nornir = value - @property - def hostname(self): - """String used to connect to the device. Either ``hostname`` or ``self.name``""" - return self.get("hostname", self.name) - - @hostname.setter - def hostname(self, value): - self.data["hostname"] = value - - @property - def port(self): - """Either ``port`` or ``None``.""" - return self.get("port") - - @port.setter - def port(self, value): - self.data["port"] = value - - @property - def username(self): - """Either ``username`` or user running the script.""" - return self.get("username", getpass.getuser()) - - @username.setter - def username(self, value): - self.data["username"] = value - - @property - def password(self): - """Either ``password`` or empty string.""" - return self.get("password", "") - - @password.setter - def password(self, value): - self.data["password"] = value - - @property - def platform(self): - """OS the device is running. Defaults to ``platform``.""" - return self.get("platform") - - @platform.setter - def platform(self, value): - self.data["platform"] = value - def get_connection_parameters( self, connection: Optional[str] = None ) -> Dict[str, Any]: @@ -329,7 +264,7 @@ def open_connection( if connection in self.connections: raise ConnectionAlreadyOpen(connection) - self.connections[connection] = self.connections.get_plugin(connection)() + self.connections[connection] = self.nornir.get_connection_type(connection)() if default_to_host_attributes: conn_params = self.get_connection_parameters(connection) self.connections[connection].open( @@ -372,105 +307,23 @@ def close_connections(self) -> None: class Group(Host): - """Same as :obj:`Host`""" - - def children(self): - return { - n: h - for n, h in self.nornir.inventory.hosts.items() - if h.has_parent_group(self) - } + pass class Inventory(object): - """ - An inventory contains information about hosts and groups. - - Arguments: - hosts (dict): keys are hostnames and values are either :obj:`Host` or a dict - representing the host data. - groups (dict): keys are group names and values are either :obj:`Group` or a dict - representing the group data. - transform_function (callable): we will call this function for each host. This is useful - to manipulate host data and make it more consumable. - - Attributes: - hosts (dict): keys are hostnames and values are :obj:`Host`. - groups (dict): keys are group names and the values are :obj:`Group`. - """ + __slots__ = ("hosts", "groups", "defaults") def __init__( - self, hosts, groups=None, defaults=None, transform_function=None, nornir=None + self, + hosts: List[Host], + groups: Optional[List[Group]] = None, + defaults: Optional[ElementData] = None, ): - self._nornir = nornir - - self.defaults = defaults or {} - - self.groups: Dict[str, Group] = {} - if groups is not None: - for group_name, group_details in groups.items(): - if group_details is None: - group = Group(name=group_name, nornir=nornir) - elif isinstance(group_details, Mapping): - group = Group(name=group_name, nornir=nornir, **group_details) - elif isinstance(group_details, Group): - group = group_details - else: - raise ValueError( - f"Parsing group {group_name}: " - f"expected dict or Group object, " - f"got {type(group_details)} instead" - ) - - self.groups[group_name] = group - - for group in self.groups.values(): - group.groups = self._resolve_groups(group.groups) - - self.hosts = {} - for n, h in hosts.items(): - if isinstance(h, Mapping): - h = Host(name=n, nornir=nornir, defaults=self.defaults, **h) - - if transform_function: - transform_function(h) - - h.groups = self._resolve_groups(h.groups) - self.hosts[n] = h - - def _resolve_groups(self, groups): - r = [] - if len(groups): - if not isinstance(groups[0], Group): - r = [self.groups[g] for g in groups] - else: - r = groups - return r + self.hosts = hosts + self.groups = groups or [] + self.defaults = defaults or ElementData() def filter(self, filter_obj=None, filter_func=None, *args, **kwargs): - """ - Returns a new inventory after filtering the hosts by matching the data passed to the - function. For instance, assume an inventory with:: - - --- - host1: - site: bma - role: http - host2: - site: cmh - role: http - host3: - site: bma - role: db - - * ``my_inventory.filter(site="bma")`` will result in ``host1`` and ``host3`` - * ``my_inventory.filter(site="bma", role="db")`` will result in ``host3`` only - - Arguments: - filter_obj (:obj:nornir.core.filter.F): Filter object to run - filter_func (callable): if filter_func is passed it will be called against each - device. If the call returns ``True`` the device will be kept in the inventory - """ filter_func = filter_obj or filter_func if filter_func: filtered = {n: h for n, h in self.hosts.items() if filter_func(h, **kwargs)} @@ -480,32 +333,7 @@ def filter(self, filter_obj=None, filter_func=None, *args, **kwargs): for n, h in self.hosts.items() if all(h.get(k) == v for k, v in kwargs.items()) } - return Inventory(hosts=filtered, groups=self.groups, nornir=self.nornir) + return Inventory(hosts=filtered, groups=self.groups) def __len__(self): return self.hosts.__len__() - - @property - def nornir(self): - """Reference to the parent :obj:`nornir.core.Nornir` object""" - return self._nornir - - @nornir.setter - def nornir(self, value): - if not getattr(self, "_nornir", None): - self._nornir = value - - for h in self.hosts.values(): - h.nornir = value - - for g in self.groups.values(): - g.nornir = value - - def to_dict(self): - """ Return a dictionary representing the object. """ - groups = {k: v.to_dict() for k, v in self.groups.items()} - groups["defaults"] = self.defaults - return { - "hosts": {k: v.to_dict() for k, v in self.hosts.items()}, - "groups": groups, - } diff --git a/nornir/core/old_inventory.py b/nornir/core/old_inventory.py new file mode 100644 index 00000000..4330e869 --- /dev/null +++ b/nornir/core/old_inventory.py @@ -0,0 +1,511 @@ +import getpass +from collections import Mapping +from typing import Any, Dict, Optional + +from nornir.core.configuration import Config +from nornir.core.connections import Connections +from nornir.core.exceptions import ConnectionAlreadyOpen, ConnectionNotOpen + + +VarsDict = Dict[str, Any] +HostsDict = Dict[str, VarsDict] +GroupsDict = Dict[str, VarsDict] + + +class Host(object): + """ + Represents a host. + + Arguments: + name (str): Name of the host + group (:obj:`Group`, optional): Group the host belongs to + nornir (:obj:`nornir.core.Nornir`): Reference to the parent nornir object + **kwargs: Host data + + Attributes: + name (str): Name of the host + groups (list of :obj:`Group`): Groups the host belongs to + defaults (``dict``): Default values for host/group data + data (dict): data about the device + connections (``dict``): Already established connections + + Note: + + You can access the host data in two ways: + + 1. Via the ``data`` attribute - In this case you will get access + **only** to the data that belongs to the host. + 2. Via the host itself as a dict - :obj:`Host` behaves like a + dict. The difference between accessing data via the ``data`` attribute + and directly via the host itself is that the latter will also + return the data if it's available via a parent :obj:`Group`. + + For instance:: + + --- + # hosts + my_host: + ip: 1.2.3.4 + groups: [bma] + + --- + # groups + bma: + site: bma + + defaults: + domain: acme.com + + * ``my_host.data["ip"]`` will return ``1.2.3.4`` + * ``my_host["ip"]`` will return ``1.2.3.4`` + * ``my_host.data["site"]`` will ``fail`` + * ``my_host["site"]`` will return ``bma`` + * ``my_host.data["domain"]`` will ``fail`` + * ``my_host.group.data["domain"]`` will ``fail`` + * ``my_host["domain"]`` will return ``acme.com`` + * ``my_host.group["domain"]`` will return ``acme.com`` + * ``my_host.group.group.data["domain"]`` will return ``acme.com`` + """ + + def __init__(self, name, groups=None, nornir=None, defaults=None, **kwargs): + self.nornir = nornir + self.name = name + self.groups = groups if groups is not None else [] + self.data = {} + self.data["name"] = name + self.connections = Connections() + self.defaults = defaults if defaults is not None else {} + + if len(self.groups): + if isinstance(groups[0], str): + self.data["groups"] = groups + else: + self.data["groups"] = [g.name for g in groups] + + for k, v in kwargs.items(): + self.data[k] = v + + def _resolve_data(self): + processed = [] + result = {} + for k, v in self.data.items(): + processed.append(k) + result[k] = v + for g in self.groups: + for k, v in g.items(): + if k not in processed: + processed.append(k) + result[k] = v + for k, v in self.defaults.items(): + if k not in processed: + processed.append(k) + result[k] = v + return result + + def keys(self): + """Returns the keys of the attribute ``data`` and of the parent(s) groups.""" + return self._resolve_data().keys() + + def values(self): + """Returns the values of the attribute ``data`` and of the parent(s) groups.""" + return self._resolve_data().values() + + def items(self): + """ + Returns all the data accessible from a device, including + the one inherited from parent groups + """ + return self._resolve_data().items() + + def to_dict(self): + """ Return a dictionary representing the object. """ + return self.data + + def has_parent_group(self, group): + """Retuns whether the object is a child of the :obj:`Group` ``group``""" + if isinstance(group, str): + return self._has_parent_group_by_name(group) + + else: + return self._has_parent_group_by_object(group) + + def _has_parent_group_by_name(self, group): + for g in self.groups: + if g.name == group or g.has_parent_group(group): + return True + + def _has_parent_group_by_object(self, group): + for g in self.groups: + if g is group or g.has_parent_group(group): + return True + + def __getitem__(self, item): + try: + return self.data[item] + + except KeyError: + for g in self.groups: + r = g.get(item) + if r: + return r + + r = self.defaults.get(item) + if r: + return r + + raise + + def __setitem__(self, item, value): + self.data[item] = value + + def __len__(self): + return len(self.keys()) + + def __iter__(self): + return self.data.__iter__() + + def __str__(self): + return self.name + + def __repr__(self): + return "{}: {}".format(self.__class__.__name__, self.name) + + def get(self, item, default=None): + """ + Returns the value ``item`` from the host or hosts group variables. + + Arguments: + item(``str``): The variable to get + default(``any``): Return value if item not found + """ + try: + return self.__getitem__(item) + + except KeyError: + return default + + @property + def nornir(self): + """Reference to the parent :obj:`nornir.core.Nornir` object""" + return self._nornir + + @nornir.setter + def nornir(self, value): + # If it's already set we don't want to set it again + # because we may lose valuable information + if not getattr(self, "_nornir", None): + self._nornir = value + + @property + def hostname(self): + """String used to connect to the device. Either ``hostname`` or ``self.name``""" + return self.get("hostname", self.name) + + @hostname.setter + def hostname(self, value): + self.data["hostname"] = value + + @property + def port(self): + """Either ``port`` or ``None``.""" + return self.get("port") + + @port.setter + def port(self, value): + self.data["port"] = value + + @property + def username(self): + """Either ``username`` or user running the script.""" + return self.get("username", getpass.getuser()) + + @username.setter + def username(self, value): + self.data["username"] = value + + @property + def password(self): + """Either ``password`` or empty string.""" + return self.get("password", "") + + @password.setter + def password(self, value): + self.data["password"] = value + + @property + def platform(self): + """OS the device is running. Defaults to ``platform``.""" + return self.get("platform") + + @platform.setter + def platform(self, value): + self.data["platform"] = value + + def get_connection_parameters( + self, connection: Optional[str] = None + ) -> Dict[str, Any]: + if not connection: + return { + "hostname": self.hostname, + "port": self.port, + "username": self.username, + "password": self.password, + "platform": self.platform, + "connection_options": {}, + } + else: + conn_params = self.get(f"{connection}_options", {}) + return { + "hostname": conn_params.get("hostname", self.hostname), + "port": conn_params.get("port", self.port), + "username": conn_params.get("username", self.username), + "password": conn_params.get("password", self.password), + "platform": conn_params.get("platform", self.platform), + "connection_options": conn_params.get("connection_options", {}), + } + + def get_connection(self, connection: str) -> Any: + """ + The function of this method is twofold: + + 1. If an existing connection is already established for the given type return it + 2. If none exists, establish a new connection of that type with default parameters + and return it + + Raises: + AttributeError: if it's unknown how to establish a connection for the given type + + Arguments: + connection: Name of the connection, for instance, netmiko, paramiko, napalm... + + Returns: + An already established connection + """ + if self.nornir: + config = self.nornir.config + else: + config = None + if connection not in self.connections: + self.open_connection( + connection, + **self.get_connection_parameters(connection), + configuration=config, + ) + return self.connections[connection].connection + + def get_connection_state(self, connection: str) -> Dict[str, Any]: + """ + For an already established connection return its state. + """ + if connection not in self.connections: + raise ConnectionNotOpen(connection) + + return self.connections[connection].state + + def open_connection( + self, + connection: str, + hostname: Optional[str] = None, + username: Optional[str] = None, + password: Optional[str] = None, + port: Optional[int] = None, + platform: Optional[str] = None, + connection_options: Optional[Dict[str, Any]] = None, + configuration: Optional[Config] = None, + default_to_host_attributes: bool = True, + ) -> None: + """ + Open a new connection. + + If ``default_to_host_attributes`` is set to ``True`` arguments will default to host + attributes if not specified. + + Raises: + AttributeError: if it's unknown how to establish a connection for the given type + + Returns: + An already established connection + """ + if connection in self.connections: + raise ConnectionAlreadyOpen(connection) + + self.connections[connection] = self.nornir.get_connection_type(connection)() + if default_to_host_attributes: + conn_params = self.get_connection_parameters(connection) + self.connections[connection].open( + hostname=hostname if hostname is not None else conn_params["hostname"], + username=username if username is not None else conn_params["username"], + password=password if password is not None else conn_params["password"], + port=port if port is not None else conn_params["port"], + platform=platform if platform is not None else conn_params["platform"], + connection_options=connection_options + if connection_options is not None + else conn_params["connection_options"], + configuration=configuration + if configuration is not None + else self.nornir.config, + ) + else: + self.connections[connection].open( + hostname=hostname, + username=username, + password=password, + port=port, + platform=platform, + connection_options=connection_options, + configuration=configuration, + ) + return self.connections[connection] + + def close_connection(self, connection: str) -> None: + """ Close the connection""" + if connection not in self.connections: + raise ConnectionNotOpen(connection) + + self.connections.pop(connection).close() + + def close_connections(self) -> None: + # Decouple deleting dictionary elements from iterating over connections dict + existing_conns = list(self.connections.keys()) + for connection in existing_conns: + self.close_connection(connection) + + +class Group(Host): + """Same as :obj:`Host`""" + + def children(self): + return { + n: h + for n, h in self.nornir.inventory.hosts.items() + if h.has_parent_group(self) + } + + +class Inventory(object): + """ + An inventory contains information about hosts and groups. + + Arguments: + hosts (dict): keys are hostnames and values are either :obj:`Host` or a dict + representing the host data. + groups (dict): keys are group names and values are either :obj:`Group` or a dict + representing the group data. + transform_function (callable): we will call this function for each host. This is useful + to manipulate host data and make it more consumable. + + Attributes: + hosts (dict): keys are hostnames and values are :obj:`Host`. + groups (dict): keys are group names and the values are :obj:`Group`. + """ + + def __init__( + self, hosts, groups=None, defaults=None, transform_function=None, nornir=None + ): + self._nornir = nornir + + self.defaults = defaults or {} + + self.groups: Dict[str, Group] = {} + if groups is not None: + for group_name, group_details in groups.items(): + if group_details is None: + group = Group(name=group_name, nornir=nornir) + elif isinstance(group_details, Mapping): + group = Group(name=group_name, nornir=nornir, **group_details) + elif isinstance(group_details, Group): + group = group_details + else: + raise ValueError( + f"Parsing group {group_name}: " + f"expected dict or Group object, " + f"got {type(group_details)} instead" + ) + + self.groups[group_name] = group + + for group in self.groups.values(): + group.groups = self._resolve_groups(group.groups) + + self.hosts = {} + for n, h in hosts.items(): + if isinstance(h, Mapping): + h = Host(name=n, nornir=nornir, defaults=self.defaults, **h) + + if transform_function: + transform_function(h) + + h.groups = self._resolve_groups(h.groups) + self.hosts[n] = h + + def _resolve_groups(self, groups): + r = [] + if len(groups): + if not isinstance(groups[0], Group): + r = [self.groups[g] for g in groups] + else: + r = groups + return r + + def filter(self, filter_obj=None, filter_func=None, *args, **kwargs): + """ + Returns a new inventory after filtering the hosts by matching the data passed to the + function. For instance, assume an inventory with:: + + --- + host1: + site: bma + role: http + host2: + site: cmh + role: http + host3: + site: bma + role: db + + * ``my_inventory.filter(site="bma")`` will result in ``host1`` and ``host3`` + * ``my_inventory.filter(site="bma", role="db")`` will result in ``host3`` only + + Arguments: + filter_obj (:obj:nornir.core.filter.F): Filter object to run + filter_func (callable): if filter_func is passed it will be called against each + device. If the call returns ``True`` the device will be kept in the inventory + """ + filter_func = filter_obj or filter_func + if filter_func: + filtered = {n: h for n, h in self.hosts.items() if filter_func(h, **kwargs)} + else: + filtered = { + n: h + for n, h in self.hosts.items() + if all(h.get(k) == v for k, v in kwargs.items()) + } + return Inventory(hosts=filtered, groups=self.groups, nornir=self.nornir) + + def __len__(self): + return self.hosts.__len__() + + @property + def nornir(self): + """Reference to the parent :obj:`nornir.core.Nornir` object""" + return self._nornir + + @nornir.setter + def nornir(self, value): + if not getattr(self, "_nornir", None): + self._nornir = value + + for h in self.hosts.values(): + h.nornir = value + + for g in self.groups.values(): + g.nornir = value + + def to_dict(self): + """ Return a dictionary representing the object. """ + groups = {k: v.to_dict() for k, v in self.groups.items()} + groups["defaults"] = self.defaults + return { + "hosts": {k: v.to_dict() for k, v in self.hosts.items()}, + "groups": groups, + } diff --git a/nornir/core/serializer.py b/nornir/core/serializer.py new file mode 100644 index 00000000..2eef0d0b --- /dev/null +++ b/nornir/core/serializer.py @@ -0,0 +1,95 @@ +from typing import Any, Dict, List, Optional + +from nornir.core.inventory import ElementData, Group, Host, Inventory + +from pydantic import BaseModel + + +class CommonAttributes(BaseModel): + hostname: Optional[str] = None + port: Optional[int] + username: Optional[str] = None + password: Optional[str] = None + platform: Optional[str] = None + data: Dict[str, Any] = {} + + class Config: + ignore_extra = False + + @staticmethod + def serialize(e: ElementData) -> "CommonAttributes": + return CommonAttributes( + hostname=e.hostname, + port=e.port, + username=e.username, + password=e.password, + platform=e.platform, + data=e.data, + ) + + +class InventoryElement(CommonAttributes): + groups: List[str] = [] + + +class HostSerializer(InventoryElement): + @staticmethod + def serialize(h: Host) -> "HostSerializer": + return HostSerializer( + hostname=object.__getattribute__(h, "hostname"), + port=object.__getattribute__(h, "port"), + username=object.__getattribute__(h, "username"), + password=object.__getattribute__(h, "password"), + platform=object.__getattribute__(h, "platform"), + groups=[c.name for c in h.groups], + data=object.__getattribute__(h, "data"), + ) + + +class GroupSerializer(InventoryElement): + def serialize(g: Group) -> "GroupSerializer": + return GroupSerializer( + hostname=object.__getattribute__(g, "hostname"), + port=object.__getattribute__(g, "port"), + username=object.__getattribute__(g, "username"), + password=object.__getattribute__(g, "password"), + platform=object.__getattribute__(g, "platform"), + groups=[c.name for c in g.groups], + data=object.__getattribute__(g, "data"), + ) + + +class InventorySerializer(BaseModel): + hosts: Dict[str, HostSerializer] + groups: Dict[str, GroupSerializer] = GroupSerializer() + defaults: CommonAttributes = CommonAttributes() + + class Config: + ignore_extra = False + + @staticmethod + def deserialize(i: Dict[str, Any]) -> Inventory: + serialized = InventorySerializer(**i) + defaults = ElementData(**serialized.defaults.dict()) + + hosts = {} + for n, h in serialized.hosts.items(): + hosts[n] = Host(name=n, **h.dict()) + groups = {} + for n, g in serialized.groups.items(): + groups[n] = Group(name=n, **g.dict()) + + for h in hosts.values(): + h.defaults = defaults + h.groups = [groups[n] for n in h.groups] + for g in groups.values(): + g.defaults = defaults + g.groups = [groups[n] for n in g.groups] + return Inventory(hosts, groups, defaults) + + @staticmethod + def serialize(i: Inventory) -> "InventorySerializer": + hosts = {n: HostSerializer.serialize(h) for n, h in i.hosts.items()} + groups = {n: GroupSerializer.serialize(g) for n, g in i.groups.items()} + defaults = CommonAttributes.serialize(i.defaults) + return InventorySerializer(hosts=hosts, groups=groups, defaults=defaults) diff --git a/nornir/plugins/inventory/simple.py b/nornir/plugins/inventory/simple.py index b77d1d43..d4ee7919 100644 --- a/nornir/plugins/inventory/simple.py +++ b/nornir/plugins/inventory/simple.py @@ -1,140 +1,26 @@ import logging import os -from typing import Any -from nornir.core.inventory import Inventory, GroupsDict, HostsDict, VarsDict +from nornir.core.inventory import Inventory import ruamel.yaml -class SimpleInventory(Inventory): - """ - This is a very simple file based inventory. Basically you need two yaml files. One - for your host information and another one for your group information. +def SimpleInventory( + host_file: str = "hosts.yaml", group_file: str = "groups.yaml" +) -> Inventory: + yml = ruamel.yaml.YAML(typ="safe") + with open(host_file, "r") as f: + hosts = yml.load(f) - * host file:: - - --- - host1.cmh: - site: cmh - role: host - groups: - - cmh-host - platform: linux - - host2.cmh: - site: cmh - role: host - groups: - - cmh-host - platform: linux - - switch00.cmh: - hostname: 127.0.0.1 - username: vagrant - password: vagrant - napalm_port: 12443 - site: cmh - role: leaf - groups: - - cmh-leaf - platform: eos - - switch01.cmh: - hostname: 127.0.0.1 - username: vagrant - password: "" - napalm_port: 12203 - site: cmh - role: leaf - groups: - - cmh-leaf - platform: juplatform - - host1.bma: - site: bma - role: host - groups: - - bma-host - platform: linux - - host2.bma: - site: bma - role: host - groups: - - bma-host - platform: linux - - switch00.bma: - hostname: 127.0.0.1 - username: vagrant - password: vagrant - napalm_port: 12443 - site: bma - role: leaf - groups: - - bma-leaf - platform: eos - - switch01.bma: - hostname: 127.0.0.1 - username: vagrant - password: "" - napalm_port: 12203 - site: bma - role: leaf - groups: - - bma-leaf - platform: juplatform - - * group file:: - - --- - defaults: - domain: acme.com - - bma-leaf: - groups: - - bma - - bma-host: - groups: - - bma - - bma: - domain: bma.acme.com - - cmh-leaf: - groups: - - cmh - - cmh-host: - groups: - - cmh - - cmh: - domain: cmh.acme.com - """ - - def __init__( - self, - host_file: str = "hosts.yaml", - group_file: str = "groups.yaml", - **kwargs: Any - ) -> None: - yml = ruamel.yaml.YAML(typ="safe") - with open(host_file, "r") as f: - hosts: HostsDict = yml.load(f) - - if group_file: - if os.path.exists(group_file): - with open(group_file, "r") as f: - groups: GroupsDict = yml.load(f) - else: - logging.warning("{}: doesn't exist".format(group_file)) - groups = {} + if group_file: + if os.path.exists(group_file): + with open(group_file, "r") as f: + groups = yml.load(f) else: + logging.warning("{}: doesn't exist".format(group_file)) groups = {} - - defaults: VarsDict = groups.pop("defaults", {}) - super().__init__(hosts, groups, defaults, **kwargs) + else: + groups = {} + defaults = groups.pop("defaults", {}) + return Inventory.from_dict(hosts, groups, defaults) diff --git a/tests/conftest.py b/tests/conftest.py index 5ba15678..3175f699 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,55 +1,55 @@ -import logging -import os -import subprocess - -from nornir.core import Nornir -from nornir.plugins.inventory.simple import SimpleInventory - -import pytest - - -logging.basicConfig( - filename="tests.log", - filemode="w", - level=logging.DEBUG, - format="%(asctime)s - %(name)s - %(levelname)s - %(funcName)20s() - %(message)s", -) - - -@pytest.fixture(scope="session", autouse=True) -def containers(request): - """Start/Stop containers needed for the tests.""" - - def fin(): - logging.info("Stopping containers") - subprocess.check_call( - ["./tests/inventory_data/containers.sh", "stop"], - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - - request.addfinalizer(fin) - - try: - fin() - except Exception: - pass - logging.info("Starting containers") - subprocess.check_call( - ["./tests/inventory_data/containers.sh", "start"], stdout=subprocess.PIPE - ) - - -@pytest.fixture(scope="session", autouse=True) -def nornir(request): - """Initializes nornir""" - dir_path = os.path.dirname(os.path.realpath(__file__)) - - nornir = Nornir( - inventory=SimpleInventory( - "{}/inventory_data/hosts.yaml".format(dir_path), - "{}/inventory_data/groups.yaml".format(dir_path), - ), - dry_run=True, - ) - return nornir +# import logging +# import os +# import subprocess + +# from nornir.core import Nornir +# from nornir.plugins.inventory.simple import SimpleInventory + +# import pytest + + +# logging.basicConfig( +# filename="tests.log", +# filemode="w", +# level=logging.DEBUG, +# format="%(asctime)s - %(name)s - %(levelname)s - %(funcName)20s() - %(message)s", +# ) + + +# @pytest.fixture(scope="session", autouse=True) +# def containers(request): +# """Start/Stop containers needed for the tests.""" + +# def fin(): +# logging.info("Stopping containers") +# subprocess.check_call( +# ["./tests/inventory_data/containers.sh", "stop"], +# stderr=subprocess.PIPE, +# stdout=subprocess.PIPE, +# ) + +# request.addfinalizer(fin) + +# try: +# fin() +# except Exception: +# pass +# logging.info("Starting containers") +# subprocess.check_call( +# ["./tests/inventory_data/containers.sh", "start"], stdout=subprocess.PIPE +# ) + + +# @pytest.fixture(scope="session", autouse=True) +# def nornir(request): +# """Initializes nornir""" +# dir_path = os.path.dirname(os.path.realpath(__file__)) + +# nornir = Nornir( +# inventory=SimpleInventory( +# "{}/inventory_data/hosts.yaml".format(dir_path), +# "{}/inventory_data/groups.yaml".format(dir_path), +# ), +# dry_run=True, +# ) +# return nornir diff --git a/tests/core/test_inventory.py b/tests/core/test_inventory.py index 03dfd87d..1f8b9c71 100644 --- a/tests/core/test_inventory.py +++ b/tests/core/test_inventory.py @@ -1,144 +1,81 @@ import os -from nornir.core.inventory import Group, Host -from nornir.plugins.inventory.simple import SimpleInventory - -import pytest - - -dir_path = os.path.dirname(os.path.realpath(__file__)) -inventory = SimpleInventory( - "{}/../inventory_data/hosts.yaml".format(dir_path), - "{}/../inventory_data/groups.yaml".format(dir_path), -) +from nornir.core.inventory import Group, Host, Inventory +from nornir.core.serializer import InventorySerializer +from pydantic import ValidationError -def compare_lists(left, right): - result = len(left) == len(right) - if not result: - return result +import pytest - def to_sets(l): - if isinstance(l, str): - return l +import ruamel.yaml - r = set() - for x in l: - if isinstance(x, list) or isinstance(x, tuple): - x = frozenset(to_sets(x)) - r.add(x) - return r - left = to_sets(left) - right = to_sets(right) - return left == right +yaml = ruamel.yaml.YAML() +dir_path = os.path.dirname(os.path.realpath(__file__)) +with open(f"{dir_path}/../inventory_data/hosts.yaml") as f: + hosts = yaml.load(f) +with open(f"{dir_path}/../inventory_data/groups.yaml") as f: + groups = yaml.load(f) +with open(f"{dir_path}/../inventory_data/defaults.yaml") as f: + defaults = yaml.load(f) +inv_dict = {"hosts": hosts, "groups": groups, "defaults": defaults} class Test(object): - def test_hosts(self): - defaults = {"var4": "ALL"} - g1 = Group(name="g1", var1="1", var2="2", var3="3") - g2 = Group(name="g2", var1="a", var2="b") - g3 = Group(name="g3", var1="A", var4="Z") - g4 = Group(name="g4", groups=[g2, g1], var3="q") - - h1 = Host(name="host1", groups=[g1, g2], defaults=defaults) - assert h1["var1"] == "1" - assert h1["var2"] == "2" - assert h1["var3"] == "3" - assert h1["var4"] == "ALL" - assert compare_lists( - list(h1.keys()), ["name", "groups", "var1", "var2", "var3", "var4"] - ) - assert compare_lists( - list(h1.values()), ["host1", ["g1", "g2"], "1", "2", "3", "ALL"] - ) - assert compare_lists( - list(h1.items()), - [ - ("name", "host1"), - ("groups", ["g1", "g2"]), - ("var1", "1"), - ("var2", "2"), - ("var3", "3"), - ("var4", "ALL"), - ], - ) - - h2 = Host(name="host2", groups=[g2, g1], defaults=defaults) - assert h2["var1"] == "a" - assert h2["var2"] == "b" - assert h2["var3"] == "3" - assert h2["var4"] == "ALL" - assert compare_lists( - list(h2.keys()), ["name", "groups", "var1", "var2", "var3", "var4"] - ) - assert compare_lists( - list(h2.values()), ["host2", ["g2", "g1"], "a", "b", "3", "ALL"] - ) - assert compare_lists( - list(h2.items()), - [ - ("name", "host2"), - ("groups", ["g2", "g1"]), - ("var1", "a"), - ("var2", "b"), - ("var3", "3"), - ("var4", "ALL"), - ], - ) - - h3 = Host(name="host3", groups=[g4, g3], defaults=defaults) - assert h3["var1"] == "a" - assert h3["var2"] == "b" - assert h3["var3"] == "q" - assert h3["var4"] == "Z" - assert compare_lists( - list(h3.keys()), ["name", "groups", "var3", "var1", "var2", "var4"] - ) - assert compare_lists( - list(h3.values()), ["host3", ["g4", "g3"], "q", "a", "b", "Z"] - ) - assert compare_lists( - list(h3.items()), - [ - ("name", "host3"), - ("groups", ["g4", "g3"]), - ("var3", "q"), - ("var1", "a"), - ("var2", "b"), - ("var4", "Z"), - ], - ) - - h4 = Host(name="host4", groups=[g3, g4], defaults=defaults) - assert h4["var1"] == "A" - assert h4["var2"] == "b" - assert h4["var3"] == "q" - assert h4["var4"] == "Z" - assert compare_lists( - list(h4.keys()), ["name", "groups", "var1", "var4", "var3", "var2"] - ) - assert compare_lists( - list(h4.values()), ["host4", ["g3", "g4"], "A", "Z", "q", "b"] - ) - assert compare_lists( - list(h4.items()), - [ - ("name", "host4"), - ("groups", ["g3", "g4"]), - ("var1", "A"), - ("var4", "Z"), - ("var3", "q"), - ("var2", "b"), - ], - ) + def test_host(self): + h = Host(name="host1", hostname="host1") + assert h.hostname == "host1" + assert h.port is None + assert h.username is None + assert h.password is None + assert h.platform is None + assert h.data == {} + + data = {"asn": 65100, "router_id": "1.1.1.1"} + h = Host( + name="host2", + hostname="host2", + username="user", + port=123, + password="", + platform="fake", + data=data, + ) + assert h.hostname == "host2" + assert h.port == 123 + assert h.username == "user" + assert h.password == "" + assert h.platform == "fake" + assert h.data == data + + def test_inventory(self): + g1 = Group(name="g1") + g2 = Group(name="g2", groups=[g1]) + h1 = Host(name="h1", groups=[g1, g2]) + h2 = Host(name="h2") + hosts = {"h1": h1, "h2": h2} + groups = {"g1": g1, "g2": g2} + inventory = Inventory(hosts=hosts, groups=groups) + assert "h1" in inventory.hosts + assert "h2" in inventory.hosts + assert "g1" in inventory.groups + assert "g2" in inventory.groups + assert inventory.groups["g1"] in inventory.hosts["h1"].groups + assert inventory.groups["g1"] in inventory.groups["g2"].groups + + def test_inventory_deserializer_wrong(self): + with pytest.raises(ValidationError): + InventorySerializer.deserialize( + {"hosts": {"wrong": {"host": "should_be_hostname"}}} + ) - with pytest.raises(KeyError): - assert h2["asdasd"] + def test_inventory_deserializer(self): + inv = InventorySerializer.deserialize(inv_dict) + assert inv.groups["group_1"] in inv.hosts["dev1.group_1"].groups def test_filtering(self): - unfiltered = sorted(list(inventory.hosts.keys())) + inv = InventorySerializer.deserialize(inv_dict) + unfiltered = sorted(list(inv.hosts.keys())) assert unfiltered == [ "dev1.group_1", "dev2.group_1", @@ -146,26 +83,21 @@ def test_filtering(self): "dev4.group_2", ] - www = sorted(list(inventory.filter(role="www").hosts.keys())) + www = sorted(list(inv.filter(role="www").hosts.keys())) assert www == ["dev1.group_1", "dev3.group_2"] - www_site1 = sorted( - list(inventory.filter(role="www", site="site1").hosts.keys()) - ) + www_site1 = sorted(list(inv.filter(role="www", site="site1").hosts.keys())) assert www_site1 == ["dev1.group_1"] www_site1 = sorted( - list(inventory.filter(role="www").filter(site="site1").hosts.keys()) + list(inv.filter(role="www").filter(site="site1").hosts.keys()) ) assert www_site1 == ["dev1.group_1"] def test_filtering_func(self): + inv = InventorySerializer.deserialize(inv_dict) long_names = sorted( - list( - inventory.filter( - filter_func=lambda x: len(x["my_var"]) > 20 - ).hosts.keys() - ) + list(inv.filter(filter_func=lambda x: len(x["my_var"]) > 20).hosts.keys()) ) assert long_names == ["dev1.group_1", "dev4.group_2"] @@ -173,107 +105,39 @@ def longer_than(dev, length): return len(dev["my_var"]) > length long_names = sorted( - list(inventory.filter(filter_func=longer_than, length=20).hosts.keys()) + list(inv.filter(filter_func=longer_than, length=20).hosts.keys()) ) assert long_names == ["dev1.group_1", "dev4.group_2"] def test_filter_unique_keys(self): - filtered = sorted(list(inventory.filter(www_server="nginx").hosts.keys())) + inv = InventorySerializer.deserialize(inv_dict) + filtered = sorted(list(inv.filter(www_server="nginx").hosts.keys())) assert filtered == ["dev1.group_1"] def test_var_resolution(self): - assert inventory.hosts["dev1.group_1"]["my_var"] == "comes_from_dev1.group_1" - assert inventory.hosts["dev2.group_1"]["my_var"] == "comes_from_group_1" - assert inventory.hosts["dev3.group_2"]["my_var"] == "comes_from_defaults" - assert inventory.hosts["dev4.group_2"]["my_var"] == "comes_from_dev4.group_2" + inv = InventorySerializer.deserialize(inv_dict) + assert inv.hosts["dev1.group_1"]["my_var"] == "comes_from_dev1.group_1" + assert inv.hosts["dev2.group_1"]["my_var"] == "comes_from_group_1" + assert inv.hosts["dev3.group_2"]["my_var"] == "comes_from_defaults" + assert inv.hosts["dev4.group_2"]["my_var"] == "comes_from_dev4.group_2" - assert ( - inventory.hosts["dev1.group_1"].data["my_var"] == "comes_from_dev1.group_1" - ) + assert inv.hosts["dev1.group_1"].data["my_var"] == "comes_from_dev1.group_1" with pytest.raises(KeyError): - inventory.hosts["dev2.group_1"].data["my_var"] + inv.hosts["dev2.group_1"].data["my_var"] with pytest.raises(KeyError): - inventory.hosts["dev3.group_2"].data["my_var"] - assert ( - inventory.hosts["dev4.group_2"].data["my_var"] == "comes_from_dev4.group_2" - ) + inv.hosts["dev3.group_2"].data["my_var"] + assert inv.hosts["dev4.group_2"].data["my_var"] == "comes_from_dev4.group_2" + + assert inv.hosts["dev1.group_1"].password == "a_password" + assert inv.hosts["dev2.group_1"].password == "docker" def test_has_parents(self): - assert inventory.hosts["dev1.group_1"].has_parent_group( - inventory.groups["group_1"] - ) - assert not inventory.hosts["dev1.group_1"].has_parent_group( - inventory.groups["group_2"] - ) - assert inventory.hosts["dev1.group_1"].has_parent_group("group_1") - assert not inventory.hosts["dev1.group_1"].has_parent_group("group_2") + inv = InventorySerializer.deserialize(inv_dict) + assert inv.hosts["dev1.group_1"].has_parent_group(inv.groups["group_1"]) + assert not inv.hosts["dev1.group_1"].has_parent_group(inv.groups["group_2"]) + assert inv.hosts["dev1.group_1"].has_parent_group("group_1") + assert not inv.hosts["dev1.group_1"].has_parent_group("group_2") def test_to_dict(self): - expected = { - "hosts": { - "dev1.group_1": { - "name": "dev1.group_1", - "nested_data": { - "a_dict": {"a": 1, "b": 2}, - "a_list": [1, 2], - "a_string": "asdasd", - }, - "groups": ["group_1"], - "my_var": "comes_from_dev1.group_1", - "www_server": "nginx", - "role": "www", - "port": 65001, - "platform": "eos", - }, - "dev3.group_2": { - "name": "dev3.group_2", - "groups": ["group_2"], - "www_server": "apache", - "role": "www", - "port": 65003, - "platform": "linux", - "napalm_options": {"platform": "mock"}, - }, - }, - "groups": { - "defaults": {}, - "parent_group": {"a_var": "blah", "name": "parent_group"}, - "group_1": { - "name": "group_1", - "my_var": "comes_from_group_1", - "site": "site1", - "groups": ["parent_group"], - }, - "group_2": {"name": "group_2", "site": "site2"}, - "empty_group": {"name": "empty_group"}, - }, - } - assert inventory.filter(role="www").to_dict() == expected - - def test_setters(self): - """Test explicit setters specified in inventory.""" - defaults = {} - g1 = Group(name="g1") - h1 = Host(name="host1", groups=[g1], defaults=defaults) - - g1.hostname = "group_hostname" - assert h1.hostname == "group_hostname" - g1.platform = "group_platform" - assert h1.platform == "group_platform" - g1.username = "group_username" - assert h1.username == "group_username" - g1.password = "group_password" - assert h1.password == "group_password" - g1.port = 9999 - assert h1.port == 9999 - - h1.hostname = "alt_hostname" - assert h1.hostname == "alt_hostname" - h1.platform = "alt_platform" - assert h1.platform == "alt_platform" - h1.username = "alt_username" - assert h1.username == "alt_username" - h1.password = "alt_password" - assert h1.password == "alt_password" - h1.port = 9998 - assert h1.port == 9998 + inv = InventorySerializer.deserialize(inv_dict) + assert InventorySerializer.serialize(inv).dict() == inv_dict diff --git a/tests/inventory_data/defaults.yaml b/tests/inventory_data/defaults.yaml new file mode 100644 index 00000000..869df583 --- /dev/null +++ b/tests/inventory_data/defaults.yaml @@ -0,0 +1,8 @@ +--- +port: +hostname: 127.0.0.1 +username: root +password: docker +platform: linux +data: + my_var: comes_from_defaults diff --git a/tests/inventory_data/groups.yaml b/tests/inventory_data/groups.yaml index 7200d86d..87b54e35 100644 --- a/tests/inventory_data/groups.yaml +++ b/tests/inventory_data/groups.yaml @@ -1,21 +1,30 @@ --- -defaults: - my_var: comes_from_defaults - hostname: 127.0.0.1 - username: root - password: docker - platform: linux - parent_group: - a_var: blah - + port: + hostname: + username: + password: + platform: + data: + a_var: blah + groups: [] group_1: - my_var: comes_from_group_1 - site: site1 + port: + hostname: + username: + password: + platform: + data: + my_var: comes_from_group_1 + site: site1 groups: - - parent_group - + - parent_group group_2: - site: site2 - -empty_group: + port: + hostname: + username: + password: + platform: + data: + site: site2 + groups: [] diff --git a/tests/inventory_data/hosts.yaml b/tests/inventory_data/hosts.yaml index 89f70112..f66995aa 100644 --- a/tests/inventory_data/hosts.yaml +++ b/tests/inventory_data/hosts.yaml @@ -1,53 +1,57 @@ --- dev1.group_1: - groups: - - group_1 - my_var: comes_from_dev1.group_1 - www_server: nginx - role: www port: 65001 + hostname: + username: + password: a_password platform: eos - nested_data: - a_dict: - a: 1 - b: 2 - a_list: [1, 2] - a_string: "asdasd" - -dev2.group_1: + data: + my_var: comes_from_dev1.group_1 + www_server: nginx + role: www + nested_data: + a_dict: + a: 1 + b: 2 + a_list: [1, 2] + a_string: asdasd groups: - group_1 - role: db +dev2.group_1: port: 65002 + hostname: + username: + password: platform: junos - nested_data: - a_dict: - b: 2 - c: 3 - a_list: [2, 3] - a_string: "qwe" - dummy_options: - hostname: overriden_hostname - port: null - connection_options: - awesome_feature: 1 - -dev3.group_2: + data: + role: db + nested_data: + a_dict: + b: 2 + c: 3 + a_list: [2, 3] + a_string: qwe groups: - - group_2 - www_server: apache - role: www + - group_1 +dev3.group_2: port: 65003 + hostname: + username: + password: platform: linux - napalm_options: - platform: mock - -dev4.group_2: + data: + www_server: apache + role: www groups: - group_2 - my_var: comes_from_dev4.group_2 - role: db +dev4.group_2: port: 65004 + hostname: + username: + password: platform: linux - napalm_options: - platform: mock + data: + my_var: comes_from_dev4.group_2 + role: db + groups: + - group_2 From 333dbd3998c6f200e5c5487683e870115e9e697d Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 2 Sep 2018 16:12:37 +0200 Subject: [PATCH 046/109] tests passing --- nornir/core/__init__.py | 30 ++- nornir/core/configuration.py | 4 + nornir/core/inventory.py | 184 ++++++++++-------- nornir/core/serializer.py | 49 +++-- nornir/plugins/connections/napalm.py | 8 +- nornir/plugins/connections/netmiko.py | 8 +- nornir/plugins/connections/paramiko.py | 10 +- nornir/plugins/inventory/ansible.py | 18 +- nornir/plugins/inventory/netbox.py | 121 ++++++------ nornir/plugins/inventory/nsot.py | 124 ++++++------ nornir/plugins/inventory/simple.py | 24 ++- nornir/plugins/tasks/text/template_file.py | 5 +- nornir/plugins/tasks/text/template_string.py | 5 +- tests/conftest.py | 112 +++++------ tests/core/test_InitNornir.py | 24 ++- tests/core/test_connections.py | 22 +-- tests/core/test_inventory.py | 45 ++++- tests/inventory_data/defaults.yaml | 11 +- tests/inventory_data/groups.yaml | 11 ++ tests/inventory_data/hosts.yaml | 26 +++ .../inventory/netbox/2.3.5/expected.json | 101 ++++++---- .../2.3.5/expected_transform_function.json | 104 ++++++---- tests/plugins/inventory/test_netbox.py | 11 +- .../tasks/networking/test_napalm_cli.py | 8 +- .../tasks/networking/test_napalm_configure.py | 14 +- .../tasks/networking/test_napalm_get.py | 18 +- .../tasks/networking/test_napalm_validate.py | 12 +- .../plugins/tasks/text/test_template_file.py | 4 +- .../tasks/text/test_template_string.py | 2 +- 29 files changed, 657 insertions(+), 458 deletions(-) diff --git a/nornir/core/__init__.py b/nornir/core/__init__.py index f036ffc6..5eeae09d 100644 --- a/nornir/core/__init__.py +++ b/nornir/core/__init__.py @@ -54,19 +54,18 @@ class Nornir(object): """ def __init__( - self, inventory, dry_run, config=None, config_file=None, logger=None, data=None + self, inventory, dry_run, _config=None, config_file=None, logger=None, data=None ): self.logger = logger or logging.getLogger("nornir") self.data = data or Data() self.inventory = inventory - self.inventory.nornir = self self.data.dry_run = dry_run if config_file: - self.config = Config(config_file=config_file) + self._config = Config(config_file=config_file) else: - self.config = config or Config() + self._config = _config or Config() def __enter__(self): return self @@ -74,6 +73,15 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self.close_connections(on_good=True, on_failed=True) + @property + def config(self): + return self._config + + @config.setter + def config(self, value): + self._config = value + self.inventory.config = value + @property def dry_run(self): return self.data.dry_run @@ -117,7 +125,7 @@ def run( raise_on_error=None, on_good=True, on_failed=False, - **kwargs + **kwargs, ): """ Run task over all the hosts in the inventory. @@ -181,6 +189,16 @@ def close_connections_task(task): self.run(task=close_connections_task, on_good=on_good, on_failed=on_failed) + @classmethod + def get_validators(cls): + yield cls.validate + + @classmethod + def validate(cls, v): + if not isinstance(v, cls): + raise ValueError(f"Nornir: Nornir expected not {type(v)}") + return v + def InitNornir(config_file="", dry_run=False, configure_logging=True, **kwargs): """ @@ -201,4 +219,4 @@ def InitNornir(config_file="", dry_run=False, configure_logging=True, **kwargs): transform_function = conf.inventory.get_transform_function() inv = inv_class(transform_function=transform_function, **conf.inventory.options) - return Nornir(inventory=inv, dry_run=dry_run, config=conf) + return Nornir(inventory=inv, dry_run=dry_run, _config=conf) diff --git a/nornir/core/configuration.py b/nornir/core/configuration.py index e08128b3..55c4b32d 100644 --- a/nornir/core/configuration.py +++ b/nornir/core/configuration.py @@ -18,6 +18,7 @@ class SSHConfig(BaseSettings): class Config: env_prefix = "NORNIR_SSH_" + ignore_extra = False class Inventory(BaseSettings): @@ -41,6 +42,7 @@ def get_transform_function(self) -> Optional[Callable[..., Any]]: class Config: env_prefix = "NORNIR_INVENTORY_" + ignore_extra = False class Logging(BaseSettings): @@ -52,6 +54,7 @@ class Logging(BaseSettings): class Config: env_prefix = "NORNIR_LOGGING_" + ignore_extra = False def configure(self): rootHandlers: List[Any] = [] @@ -123,6 +126,7 @@ class Config(BaseSettings): class Config: env_prefix = "NORNIR_" + ignore_extra = False def __init__(self, path: str = "", **kwargs) -> None: if path: diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index d3116484..ccd48aaf 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -4,52 +4,44 @@ from nornir.core.connections import Connections from nornir.core.exceptions import ConnectionAlreadyOpen, ConnectionNotOpen +from pydantic import BaseModel + GroupsDict = None # DELETEME HostsDict = None # DELETEME VarsDict = None # DELETEME -class ElementData(object): - __slots__ = ( - "hostname", - "port", - "username", - "password", - "platform", - "groups", - "data", - "connections", - ) +class ConnectionOptions(BaseModel): + hostname: Optional[str] = None + port: Optional[int] = None + username: Optional[str] = None + password: Optional[str] = None + platform: Optional[str] = None + extras: Dict[str, Any] = {} - def __init__( - self, - hostname: Optional[str] = None, - port: Optional[str] = None, - username: Optional[str] = None, - password: Optional[str] = None, - platform: Optional[str] = None, - groups: Optional[List["Group"]] = None, - data: Optional[Dict[str, Any]] = None, - ) -> None: - self.hostname = hostname - self.port = port - self.username = username - self.password = password - self.platform = platform - self.groups = groups or [] - self.data = data or {} - self.connections = Connections() + class Config: + ignore_extra = False -class Host(ElementData): - __slots__ = ("name", "defaults") +class Groups(List["Group"]): + def __contains__(self, value) -> bool: + if isinstance(value, str): + return any([g.name == value for g in self]) + else: + return any([g is value for g in self]) - def __init__( - self, name: str, defaults: Optional[ElementData] = None, *args, **kwargs - ) -> None: - self.name = name - self.defaults = defaults or ElementData() - super().__init__(*args, **kwargs) + +class ElementData(ConnectionOptions): + groups: Groups = Groups() + data: Dict[str, Any] = {} + connection_options: Dict[str, ConnectionOptions] = {} + + +class Host(ElementData): + name: str + defaults: ElementData = {} + connections: Connections = Connections() + _config: Config = Config() def _resolve_data(self): processed = [] @@ -124,14 +116,14 @@ def __getitem__(self, item): def __getattribute__(self, name): if name not in ("hostname", "port", "username", "password", "platform"): return object.__getattribute__(self, name) - v = object.__getattribute__(self, name) + v = self.__values__[name] if v is None: for g in self.groups: - r = object.__getattribute__(g, name) + r = g.__values__[name] if r is not None: return r - return getattr(self.defaults, name) + return self.defaults.__values__[name] else: return v @@ -158,46 +150,52 @@ def get(self, item, default=None): item(``str``): The variable to get default(``any``): Return value if item not found """ + if hasattr(self, item): + return getattr(self, item) try: return self.__getitem__(item) except KeyError: return default - @property - def nornir(self): - """Reference to the parent :obj:`nornir.core.Nornir` object""" - return self._nornir - - @nornir.setter - def nornir(self, value): - # If it's already set we don't want to set it again - # because we may lose valuable information - if not getattr(self, "_nornir", None): - self._nornir = value - def get_connection_parameters( self, connection: Optional[str] = None ) -> Dict[str, Any]: if not connection: - return { + d = { "hostname": self.hostname, "port": self.port, "username": self.username, "password": self.password, "platform": self.platform, - "connection_options": {}, + "extras": {}, } else: - conn_params = self.get(f"{connection}_options", {}) - return { - "hostname": conn_params.get("hostname", self.hostname), - "port": conn_params.get("port", self.port), - "username": conn_params.get("username", self.username), - "password": conn_params.get("password", self.password), - "platform": conn_params.get("platform", self.platform), - "connection_options": conn_params.get("connection_options", {}), - } + d = self._get_connection_options_recursively(connection) + if d is not None: + return d + else: + d = { + "hostname": self.hostname, + "port": self.port, + "username": self.username, + "password": self.password, + "platform": self.platform, + "extras": {}, + } + return ConnectionOptions(**d) + + def _get_connection_options_recursively(self, connection: str) -> Dict[str, Any]: + p = self.connection_options.get(connection) + if p is None: + for g in self.groups: + p = g._get_connection_options_recursively(connection) + if p is not None: + return p + + return self.defaults.connection_options.get(connection, None) + else: + return p def get_connection(self, connection: str) -> Any: """ @@ -216,15 +214,11 @@ def get_connection(self, connection: str) -> Any: Returns: An already established connection """ - if self.nornir: - config = self.nornir.config - else: - config = None if connection not in self.connections: self.open_connection( connection, - **self.get_connection_parameters(connection), - configuration=config, + **self.get_connection_parameters(connection).dict(), + configuration=self.config, ) return self.connections[connection].connection @@ -245,7 +239,7 @@ def open_connection( password: Optional[str] = None, port: Optional[int] = None, platform: Optional[str] = None, - connection_options: Optional[Dict[str, Any]] = None, + extras: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, default_to_host_attributes: bool = True, ) -> None: @@ -264,21 +258,19 @@ def open_connection( if connection in self.connections: raise ConnectionAlreadyOpen(connection) - self.connections[connection] = self.nornir.get_connection_type(connection)() + self.connections[connection] = self.connections.get_plugin(connection)() if default_to_host_attributes: conn_params = self.get_connection_parameters(connection) self.connections[connection].open( - hostname=hostname if hostname is not None else conn_params["hostname"], - username=username if username is not None else conn_params["username"], - password=password if password is not None else conn_params["password"], - port=port if port is not None else conn_params["port"], - platform=platform if platform is not None else conn_params["platform"], - connection_options=connection_options - if connection_options is not None - else conn_params["connection_options"], + hostname=hostname if hostname is not None else conn_params.hostname, + username=username if username is not None else conn_params.username, + password=password if password is not None else conn_params.password, + port=port if port is not None else conn_params.port, + platform=platform if platform is not None else conn_params.platform, + extras=extras if extras is not None else conn_params.extras, configuration=configuration if configuration is not None - else self.nornir.config, + else self.config, ) else: self.connections[connection].open( @@ -287,7 +279,7 @@ def open_connection( password=password, port=port, platform=platform, - connection_options=connection_options, + extras=extras, configuration=configuration, ) return self.connections[connection] @@ -305,24 +297,40 @@ def close_connections(self) -> None: for connection in existing_conns: self.close_connection(connection) + @property + def config(self): + return self._config + + @config.setter + def config(self, value): + self._config = value + class Group(Host): pass +# TODO use basemodel class Inventory(object): - __slots__ = ("hosts", "groups", "defaults") + __slots__ = ("hosts", "groups", "defaults", "_nornir", "_config") def __init__( self, hosts: List[Host], groups: Optional[List[Group]] = None, defaults: Optional[ElementData] = None, + config: Optional[Config] = None, + transform_function=None, ): + self._config = config self.hosts = hosts - self.groups = groups or [] + self.groups = groups or {} self.defaults = defaults or ElementData() + if transform_function: + for h in self.hosts.values(): + transform_function(h) + def filter(self, filter_obj=None, filter_func=None, *args, **kwargs): filter_func = filter_obj or filter_func if filter_func: @@ -337,3 +345,13 @@ def filter(self, filter_obj=None, filter_func=None, *args, **kwargs): def __len__(self): return self.hosts.__len__() + + @property + def config(self): + return self._config + + @config.setter + def config(self, value): + self._config = value + for host in self.hosts.values(): + host._config = value diff --git a/nornir/core/serializer.py b/nornir/core/serializer.py index 2eef0d0b..67a99684 100644 --- a/nornir/core/serializer.py +++ b/nornir/core/serializer.py @@ -1,21 +1,29 @@ from typing import Any, Dict, List, Optional -from nornir.core.inventory import ElementData, Group, Host, Inventory +from nornir.core.inventory import ElementData, Group, Groups, Host, Inventory from pydantic import BaseModel -class CommonAttributes(BaseModel): +class BaseAttributes(BaseModel): hostname: Optional[str] = None port: Optional[int] username: Optional[str] = None password: Optional[str] = None platform: Optional[str] = None - data: Dict[str, Any] = {} class Config: ignore_extra = False + +class ConnectionOptions(BaseAttributes): + extras: Dict[str, Any] = {} + + +class CommonAttributes(BaseAttributes): + data: Dict[str, Any] = {} + connection_options: Dict[str, ConnectionOptions] = {} + @staticmethod def serialize(e: ElementData) -> "CommonAttributes": return CommonAttributes( @@ -25,6 +33,7 @@ def serialize(e: ElementData) -> "CommonAttributes": password=e.password, platform=e.platform, data=e.data, + connection_options=e.connection_options, ) @@ -36,26 +45,28 @@ class HostSerializer(InventoryElement): @staticmethod def serialize(h: Host) -> "HostSerializer": return HostSerializer( - hostname=object.__getattribute__(h, "hostname"), - port=object.__getattribute__(h, "port"), - username=object.__getattribute__(h, "username"), - password=object.__getattribute__(h, "password"), - platform=object.__getattribute__(h, "platform"), + hostname=h.__values__["hostname"], + port=h.__values__["port"], + username=h.__values__["username"], + password=h.__values__["password"], + platform=h.__values__["platform"], groups=[c.name for c in h.groups], - data=object.__getattribute__(h, "data"), + data=h.__values__["data"], + connection_options=h.__values__["connection_options"], ) class GroupSerializer(InventoryElement): def serialize(g: Group) -> "GroupSerializer": return GroupSerializer( - hostname=object.__getattribute__(g, "hostname"), - port=object.__getattribute__(g, "port"), - username=object.__getattribute__(g, "username"), - password=object.__getattribute__(g, "password"), - platform=object.__getattribute__(g, "platform"), + hostname=g.__values__["hostname"], + port=g.__values__["port"], + username=g.__values__["username"], + password=g.__values__["password"], + platform=g.__values__["platform"], groups=[c.name for c in g.groups], - data=object.__getattribute__(g, "data"), + data=g.__values__["data"], + connection_options=g.__values__["connection_options"], ) @@ -68,7 +79,7 @@ class Config: ignore_extra = False @staticmethod - def deserialize(i: Dict[str, Any]) -> Inventory: + def deserialize(i: Dict[str, Any], *args, **kwargs) -> Inventory: serialized = InventorySerializer(**i) defaults = ElementData(**serialized.defaults.dict()) @@ -81,11 +92,11 @@ def deserialize(i: Dict[str, Any]) -> Inventory: for h in hosts.values(): h.defaults = defaults - h.groups = [groups[n] for n in h.groups] + h.groups = Groups([groups[n] for n in h.groups]) for g in groups.values(): g.defaults = defaults - g.groups = [groups[n] for n in g.groups] - return Inventory(hosts, groups, defaults) + g.groups = Groups([groups[n] for n in g.groups]) + return Inventory(hosts, groups, defaults, *args, **kwargs) @staticmethod def serialize(i: Inventory) -> "InventorySerializer": diff --git a/nornir/plugins/connections/napalm.py b/nornir/plugins/connections/napalm.py index d7a78ae2..36c625e2 100644 --- a/nornir/plugins/connections/napalm.py +++ b/nornir/plugins/connections/napalm.py @@ -12,7 +12,7 @@ class Napalm(ConnectionPlugin): relevant connection. Inventory: - connection_options: passed as it is to the napalm driver + extras: passed as it is to the napalm driver """ def open( @@ -22,10 +22,10 @@ def open( password: Optional[str], port: Optional[int], platform: Optional[str], - connection_options: Optional[Dict[str, Any]] = None, + extras: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: - connection_options = connection_options or {} + extras = extras or {} parameters: Dict[str, Any] = { "hostname": hostname, @@ -33,7 +33,7 @@ def open( "password": password, "optional_args": {}, } - parameters.update(connection_options) + parameters.update(extras) if port and "port" not in parameters["optional_args"]: parameters["optional_args"]["port"] = port diff --git a/nornir/plugins/connections/netmiko.py b/nornir/plugins/connections/netmiko.py index 8dfdc1d4..58801ffe 100644 --- a/nornir/plugins/connections/netmiko.py +++ b/nornir/plugins/connections/netmiko.py @@ -20,7 +20,7 @@ class Netmiko(ConnectionPlugin): relevant connection. Inventory: - connection_options: maps to argument passed to ``ConnectHandler``. + extras: maps to argument passed to ``ConnectHandler``. """ def open( @@ -30,7 +30,7 @@ def open( password: Optional[str], port: Optional[int], platform: Optional[str], - connection_options: Optional[Dict[str, Any]] = None, + extras: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: parameters = { @@ -45,8 +45,8 @@ def open( platform = napalm_to_netmiko_map.get(platform, platform) parameters["device_type"] = platform - connection_options = connection_options or {} - parameters.update(connection_options) + extras = extras or {} + parameters.update(extras) self.connection = ConnectHandler(**parameters) def close(self) -> None: diff --git a/nornir/plugins/connections/paramiko.py b/nornir/plugins/connections/paramiko.py index aa9ade1b..248a5724 100644 --- a/nornir/plugins/connections/paramiko.py +++ b/nornir/plugins/connections/paramiko.py @@ -13,7 +13,7 @@ class Paramiko(ConnectionPlugin): relevant connection. Inventory: - connection_options: maps to argument passed to ``ConnectHandler``. + extras: maps to argument passed to ``ConnectHandler``. """ def open( @@ -23,10 +23,10 @@ def open( password: Optional[str], port: Optional[int], platform: Optional[str], - connection_options: Optional[Dict[str, Any]] = None, + extras: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: - connection_options = connection_options or {} + extras = extras or {} client = paramiko.SSHClient() client._policy = paramiko.WarningPolicy() @@ -60,8 +60,8 @@ def open( if "identityfile" in user_config: parameters["key_filename"] = user_config["identityfile"] - connection_options.update(parameters) - client.connect(**connection_options) + extras.update(parameters) + client.connect(**extras) self.connection = client def close(self) -> None: diff --git a/nornir/plugins/inventory/ansible.py b/nornir/plugins/inventory/ansible.py index 867c0dde..600a0251 100644 --- a/nornir/plugins/inventory/ansible.py +++ b/nornir/plugins/inventory/ansible.py @@ -11,6 +11,7 @@ from ruamel.yaml.composer import ComposerError from nornir.core.inventory import Inventory, VarsDict, GroupsDict, HostsDict +from nornir.core.serializer import InventorySerializer VARS_FILENAME_EXTENSIONS = ["", ".yml", ".yaml"] @@ -234,15 +235,8 @@ def parse(hostsfile: str) -> Tuple[HostsDict, GroupsDict]: return parser.hosts, parser.groups -class AnsibleInventory(Inventory): - """ - Inventory plugin that is capable of reading an ansible inventory. - - Arguments: - hostsfile (string): Ansible inventory file to load - """ - - def __init__(self, hostsfile: str = "hosts", **kwargs: Any) -> None: - host_vars, group_vars = parse(hostsfile) - defaults = group_vars.pop("defaults") - super().__init__(host_vars, group_vars, defaults, **kwargs) +def AnsibleInventory(hostsfile: str = "hosts", *args: Any, **kwargs: Any) -> Inventory: + host_vars, group_vars = parse(hostsfile) + defaults = group_vars.pop("defaults") + inv_dict = {"hosts": host_vars, "groups": group_vars, "defaults": defaults} + return InventorySerializer.deserialize(inv_dict, *args, **kwargs) diff --git a/nornir/plugins/inventory/netbox.py b/nornir/plugins/inventory/netbox.py index d23d3dc1..099f3a07 100644 --- a/nornir/plugins/inventory/netbox.py +++ b/nornir/plugins/inventory/netbox.py @@ -1,70 +1,65 @@ import os -from builtins import super from nornir.core.inventory import Inventory +from nornir.core.serializer import HostSerializer, InventorySerializer import requests -class NBInventory(Inventory): - def __init__( - self, - nb_url=None, - nb_token=None, - use_slugs=True, - flatten_custom_fields=True, - **kwargs - ): - - nb_url = nb_url or os.environ.get("NB_URL", "http://localhost:8080") - nb_token = nb_token or os.environ.get( - "NB_TOKEN", "0123456789abcdef0123456789abcdef01234567" - ) - headers = {"Authorization": "Token {}".format(nb_token)} - - # Create dict of hosts using 'devices' from NetBox - r = requests.get("{}/api/dcim/devices/?limit=0".format(nb_url), headers=headers) - r.raise_for_status() - nb_devices = r.json() - - devices = {} - for d in nb_devices["results"]: - - # Create temporary dict - temp = {} - - # Add value for IP address - if d.get("primary_ip", {}): - temp["hostname"] = d["primary_ip"]["address"].split("/")[0] - - # Add values that don't have an option for 'slug' - temp["serial"] = d["serial"] - temp["vendor"] = d["device_type"]["manufacturer"]["name"] - temp["asset_tag"] = d["asset_tag"] - - if flatten_custom_fields: - for cf, value in d["custom_fields"].items(): - temp[cf] = value - else: - temp["custom_fields"] = d["custom_fields"] - - # Add values that do have an option for 'slug' - if use_slugs: - temp["site"] = d["site"]["slug"] - temp["role"] = d["device_role"]["slug"] - temp["model"] = d["device_type"]["slug"] - - # Attempt to add 'platform' based of value in 'slug' - temp["platform"] = d["platform"]["slug"] if d["platform"] else None - - else: - temp["site"] = d["site"]["name"] - temp["role"] = d["device_role"] - temp["model"] = d["device_type"] - temp["platform"] = d["platform"] - - # Assign temporary dict to outer dict - devices[d["name"]] = temp - - # Pass the data back to the parent class - super().__init__(devices, None, **kwargs) +def NBInventory( + nb_url=None, nb_token=None, use_slugs=True, flatten_custom_fields=True, **kwargs +) -> Inventory: + + nb_url = nb_url or os.environ.get("NB_URL", "http://localhost:8080") + nb_token = nb_token or os.environ.get( + "NB_TOKEN", "0123456789abcdef0123456789abcdef01234567" + ) + headers = {"Authorization": "Token {}".format(nb_token)} + + # Create dict of hosts using 'devices' from NetBox + r = requests.get("{}/api/dcim/devices/?limit=0".format(nb_url), headers=headers) + r.raise_for_status() + nb_devices = r.json() + + hosts = {} + for d in nb_devices["results"]: + + # Create temporary dict + host = HostSerializer() + + # Add value for IP address + if d.get("primary_ip", {}): + host.hostname = d["primary_ip"]["address"].split("/")[0] + + # Add values that don't have an option for 'slug' + host.data["serial"] = d["serial"] + host.data["vendor"] = d["device_type"]["manufacturer"]["name"] + host.data["asset_tag"] = d["asset_tag"] + + if flatten_custom_fields: + for cf, value in d["custom_fields"].items(): + host[cf] = value + else: + host.data["custom_fields"] = d["custom_fields"] + + # Add values that do have an option for 'slug' + if use_slugs: + host.data["site"] = d["site"]["slug"] + host.data["role"] = d["device_role"]["slug"] + host.data["model"] = d["device_type"]["slug"] + + # Attempt to add 'platform' based of value in 'slug' + host.platform = d["platform"]["slug"] if d["platform"] else None + + else: + host.data["site"] = d["site"]["name"] + host.data["role"] = d["device_role"] + host.data["model"] = d["device_type"] + host.platform = d["platform"] + + # Assign temporary dict to outer dict + hosts[d["name"]] = host + + # Pass the data back to the parent class + inv_dict = {"hosts": hosts, "groups": {}, "defaults": {}} + return InventorySerializer.deserialize(inv_dict, **kwargs) diff --git a/nornir/plugins/inventory/nsot.py b/nornir/plugins/inventory/nsot.py index 6fff8913..7b5bed1e 100644 --- a/nornir/plugins/inventory/nsot.py +++ b/nornir/plugins/inventory/nsot.py @@ -3,10 +3,19 @@ import requests -from nornir.core.inventory import Inventory, VarsDict, HostsDict - - -class NSOTInventory(Inventory): +from nornir.core.inventory import Inventory, VarsDict, HostsDict, ElementData +from nornir.core.serializer import InventorySerializer + + +def NSOTInventory( + nsot_url: str = "", + nsot_email: str = "", + nsot_secret_key: str = "", + nsot_auth_header: str = "", + flatten_attributes: bool = True, + *args: Any, + **kwargs: Any +) -> Inventory: """ Inventory plugin that uses `nsot `_ as backend. @@ -30,58 +39,55 @@ class NSOTInventory(Inventory): will be used as auth_method. """ - def __init__( - self, - nsot_url: str = "", - nsot_email: str = "", - nsot_secret_key: str = "", - nsot_auth_header: str = "", - flatten_attributes: bool = True, - **kwargs: Any - ) -> None: - nsot_url = nsot_url or os.environ.get("NSOT_URL", "http://localhost:8990/api") - nsot_email = nsot_email or os.environ.get("NSOT_EMAIL", "admin@acme.com") - secret_key = nsot_secret_key or os.environ.get("NSOT_SECRET_KEY") - - if secret_key: - data = {"email": nsot_email, "secret_key": secret_key} - res = requests.post("{}/authenticate/".format(nsot_url), data=data) - auth_token = res.json().get("auth_token") - headers = { - "Authorization": "AuthToken {}:{}".format(nsot_email, auth_token) - } - - else: - nsot_auth_header = nsot_auth_header or os.environ.get( - "NSOT_AUTH_HEADER", "X-NSoT-Email" - ) - headers = {nsot_auth_header: nsot_email} - - devices: List[VarsDict] = requests.get( - "{}/devices".format(nsot_url), headers=headers - ).json() - sites: List[VarsDict] = requests.get( - "{}/sites".format(nsot_url), headers=headers - ).json() - interfaces: List[VarsDict] = requests.get( - "{}/interfaces".format(nsot_url), headers=headers - ).json() - - # We resolve site_id and assign "site" variable with the name of the site - for d in devices: - d["site"] = sites[d["site_id"] - 1]["name"] - d["interfaces"] = {} - - if flatten_attributes: - # We assign attributes to the root - for k, v in d.pop("attributes").items(): - d[k] = v - - # We assign the interfaces to the hosts - for i in interfaces: - devices[i["device"] - 1]["interfaces"][i["name"]] = i - - # Finally the inventory expects a dict of hosts where the key is the hostname - hosts: HostsDict = {d["hostname"]: d for d in devices} - - super().__init__(hosts, None, **kwargs) + nsot_url = nsot_url or os.environ.get("NSOT_URL", "http://localhost:8990/api") + nsot_email = nsot_email or os.environ.get("NSOT_EMAIL", "admin@acme.com") + secret_key = nsot_secret_key or os.environ.get("NSOT_SECRET_KEY") + + if secret_key: + data = {"email": nsot_email, "secret_key": secret_key} + res = requests.post("{}/authenticate/".format(nsot_url), data=data) + auth_token = res.json().get("auth_token") + headers = {"Authorization": "AuthToken {}:{}".format(nsot_email, auth_token)} + + else: + nsot_auth_header = nsot_auth_header or os.environ.get( + "NSOT_AUTH_HEADER", "X-NSoT-Email" + ) + headers = {nsot_auth_header: nsot_email} + + devices: List[VarsDict] = requests.get( + "{}/devices".format(nsot_url), headers=headers + ).json() + sites: List[VarsDict] = requests.get( + "{}/sites".format(nsot_url), headers=headers + ).json() + interfaces: List[VarsDict] = requests.get( + "{}/interfaces".format(nsot_url), headers=headers + ).json() + + # We resolve site_id and assign "site" variable with the name of the site + for d in devices: + d["data"] = {"site": sites[d["site_id"] - 1]["name"], "interfaces": {}} + + remove_keys = [] + for k, v in d.items(): + if k not in ElementData().fields: + remove_keys.append(k) + d["data"][k] = v + for r in remove_keys: + d.pop(r) + + if flatten_attributes: + # We assign attributes to the root + for k, v in d["data"].pop("attributes").items(): + d["data"][k] = v + + # We assign the interfaces to the hosts + for i in interfaces: + devices[i["device"] - 1]["data"]["interfaces"][i["name"]] = i + + # Finally the inventory expects a dict of hosts where the key is the hostname + hosts: HostsDict = {d["hostname"]: d for d in devices} + + inv_dict = {"hosts": hosts, "groups": {}, "defaults": {}} + return InventorySerializer.deserialize(inv_dict, *args, **kwargs) diff --git a/nornir/plugins/inventory/simple.py b/nornir/plugins/inventory/simple.py index d4ee7919..d4784de9 100644 --- a/nornir/plugins/inventory/simple.py +++ b/nornir/plugins/inventory/simple.py @@ -2,17 +2,23 @@ import os from nornir.core.inventory import Inventory +from nornir.core.serializer import InventorySerializer import ruamel.yaml def SimpleInventory( - host_file: str = "hosts.yaml", group_file: str = "groups.yaml" + host_file: str = "hosts.yaml", + group_file: str = "groups.yaml", + defaults_file: str = "defaults.yaml", + *args, + **kwargs ) -> Inventory: yml = ruamel.yaml.YAML(typ="safe") with open(host_file, "r") as f: hosts = yml.load(f) + groups = {} if group_file: if os.path.exists(group_file): with open(group_file, "r") as f: @@ -20,7 +26,15 @@ def SimpleInventory( else: logging.warning("{}: doesn't exist".format(group_file)) groups = {} - else: - groups = {} - defaults = groups.pop("defaults", {}) - return Inventory.from_dict(hosts, groups, defaults) + + defaults = {} + if defaults_file: + if os.path.exists(defaults_file): + with open(defaults_file, "r") as f: + defaults = yml.load(f) + else: + logging.warning("{}: doesn't exist".format(defaults_file)) + defaults = {} + + inv_dict = {"hosts": hosts, "groups": groups, "defaults": defaults} + return InventorySerializer.deserialize(inv_dict, *args, **kwargs) diff --git a/nornir/plugins/tasks/text/template_file.py b/nornir/plugins/tasks/text/template_file.py index 387fa0b9..4c86962c 100644 --- a/nornir/plugins/tasks/text/template_file.py +++ b/nornir/plugins/tasks/text/template_file.py @@ -1,6 +1,6 @@ from typing import Any, Optional, Dict, Callable -from nornir.core.helpers import jinja_helper, merge_two_dicts +from nornir.core.helpers import jinja_helper from nornir.core.task import Result, Task FiltersDict = Optional[Dict[str, Callable[..., str]]] @@ -27,12 +27,11 @@ def template_file( * result (``string``): rendered string """ jinja_filters = jinja_filters or {} or task.nornir.config.jinja_filters - merged = merge_two_dicts(task.host, kwargs) text = jinja_helper.render_from_file( template=template, path=path, host=task.host, jinja_filters=jinja_filters, - **merged + **kwargs ) return Result(host=task.host, result=text) diff --git a/nornir/plugins/tasks/text/template_string.py b/nornir/plugins/tasks/text/template_string.py index cec83cc2..1e808c91 100644 --- a/nornir/plugins/tasks/text/template_string.py +++ b/nornir/plugins/tasks/text/template_string.py @@ -1,6 +1,6 @@ from typing import Any, Optional, Dict, Callable -from nornir.core.helpers import jinja_helper, merge_two_dicts +from nornir.core.helpers import jinja_helper from nornir.core.task import Result, Task FiltersDict = Optional[Dict[str, Callable[..., str]]] @@ -22,8 +22,7 @@ def template_string( * result (``string``): rendered string """ jinja_filters = jinja_filters or {} or task.nornir.config.jinja_filters - merged = merge_two_dicts(task.host, kwargs) text = jinja_helper.render_from_string( - template=template, host=task.host, jinja_filters=jinja_filters, **merged + template=template, host=task.host, jinja_filters=jinja_filters, **kwargs ) return Result(host=task.host, result=text) diff --git a/tests/conftest.py b/tests/conftest.py index 3175f699..526075b7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,55 +1,57 @@ -# import logging -# import os -# import subprocess - -# from nornir.core import Nornir -# from nornir.plugins.inventory.simple import SimpleInventory - -# import pytest - - -# logging.basicConfig( -# filename="tests.log", -# filemode="w", -# level=logging.DEBUG, -# format="%(asctime)s - %(name)s - %(levelname)s - %(funcName)20s() - %(message)s", -# ) - - -# @pytest.fixture(scope="session", autouse=True) -# def containers(request): -# """Start/Stop containers needed for the tests.""" - -# def fin(): -# logging.info("Stopping containers") -# subprocess.check_call( -# ["./tests/inventory_data/containers.sh", "stop"], -# stderr=subprocess.PIPE, -# stdout=subprocess.PIPE, -# ) - -# request.addfinalizer(fin) - -# try: -# fin() -# except Exception: -# pass -# logging.info("Starting containers") -# subprocess.check_call( -# ["./tests/inventory_data/containers.sh", "start"], stdout=subprocess.PIPE -# ) - - -# @pytest.fixture(scope="session", autouse=True) -# def nornir(request): -# """Initializes nornir""" -# dir_path = os.path.dirname(os.path.realpath(__file__)) - -# nornir = Nornir( -# inventory=SimpleInventory( -# "{}/inventory_data/hosts.yaml".format(dir_path), -# "{}/inventory_data/groups.yaml".format(dir_path), -# ), -# dry_run=True, -# ) -# return nornir +import logging +import os +import subprocess + +from nornir.core import InitNornir + +import pytest + + +logging.basicConfig( + filename="tests.log", + filemode="w", + level=logging.DEBUG, + format="%(asctime)s - %(name)s - %(levelname)s - %(funcName)20s() - %(message)s", +) + + +@pytest.fixture(scope="session", autouse=True) +def containers(request): + """Start/Stop containers needed for the tests.""" + + def fin(): + logging.info("Stopping containers") + subprocess.check_call( + ["./tests/inventory_data/containers.sh", "stop"], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + request.addfinalizer(fin) + + try: + fin() + except Exception: + pass + logging.info("Starting containers") + subprocess.check_call( + ["./tests/inventory_data/containers.sh", "start"], stdout=subprocess.PIPE + ) + + +@pytest.fixture(scope="session", autouse=True) +def nornir(request): + """Initializes nornir""" + dir_path = os.path.dirname(os.path.realpath(__file__)) + + nornir = InitNornir( + inventory={ + "options": { + "host_file": "{}/inventory_data/hosts.yaml".format(dir_path), + "group_file": "{}/inventory_data/groups.yaml".format(dir_path), + "defaults_file": "{}/inventory_data/defaults.yaml".format(dir_path), + } + }, + dry_run=True, + ) + return nornir diff --git a/tests/core/test_InitNornir.py b/tests/core/test_InitNornir.py index 195a2dae..fe7a71e5 100644 --- a/tests/core/test_InitNornir.py +++ b/tests/core/test_InitNornir.py @@ -1,22 +1,20 @@ import os -from builtins import super from nornir.core import InitNornir -from nornir.core.inventory import Inventory +from nornir.core.serializer import InventorySerializer dir_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "test_InitNornir") def transform_func(host): - host.processed_by_transform_function = True + host["processed_by_transform_function"] = True -class StringInventory(Inventory): - def __init__(self, *args, **kwargs): - hosts = {"host1": {}, "host2": {}} - super().__init__(hosts, *args, **kwargs) +def StringInventory(**kwargs): + inv_dict = {"hosts": {"host1": {}, "host2": {}}, "groups": {}, "defaults": {}} + return InventorySerializer.deserialize(inv_dict, **kwargs) class Test(object): @@ -68,14 +66,14 @@ def test_InitNornir_different_inventory_by_string(self): config_file=os.path.join(dir_path, "a_config.yaml"), inventory={"plugin": "tests.core.test_InitNornir.StringInventory"}, ) - assert isinstance(nr.inventory, StringInventory) + assert "host1" in nr.inventory.hosts def test_InitNornir_different_inventory_imported(self): nr = InitNornir( config_file=os.path.join(dir_path, "a_config.yaml"), inventory={"plugin": StringInventory}, ) - assert isinstance(nr.inventory, StringInventory) + assert "host1" in nr.inventory.hosts def test_InitNornir_different_transform_function_by_string(self): nr = InitNornir( @@ -89,8 +87,8 @@ def test_InitNornir_different_transform_function_by_string(self): }, }, ) - for value in nr.inventory.hosts.values(): - assert value.processed_by_transform_function + for host in nr.inventory.hosts.values(): + assert host["processed_by_transform_function"] def test_InitNornir_different_transform_function_imported(self): nr = InitNornir( @@ -104,5 +102,5 @@ def test_InitNornir_different_transform_function_imported(self): }, }, ) - for value in nr.inventory.hosts.values(): - assert value.processed_by_transform_function + for host in nr.inventory.hosts.values(): + assert host["processed_by_transform_function"] diff --git a/tests/core/test_connections.py b/tests/core/test_connections.py index 54c277d4..2432b239 100644 --- a/tests/core/test_connections.py +++ b/tests/core/test_connections.py @@ -1,17 +1,17 @@ from typing import Any, Dict, Optional -import pytest - from nornir.core.configuration import Config from nornir.core.connections import ConnectionPlugin, Connections from nornir.core.exceptions import ( ConnectionAlreadyOpen, ConnectionNotOpen, - ConnectionPluginNotRegistered, ConnectionPluginAlreadyRegistered, + ConnectionPluginNotRegistered, ) from nornir.plugins.connections import register_default_connection_plugins +import pytest + class DummyConnectionPlugin(ConnectionPlugin): def open( @@ -21,7 +21,7 @@ def open( password: Optional[str], port: Optional[int], platform: Optional[str], - connection_options: Optional[Dict[str, Any]] = None, + extras: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: self.connection = True @@ -31,7 +31,7 @@ def open( self.password = password self.port = port self.platform = platform - self.connection_options = connection_options + self.extras = extras self.configuration = configuration def close(self) -> None: @@ -118,7 +118,7 @@ def test_validate_params_simple(self, nornir): "password": "docker", "port": 65002, "platform": "junos", - "connection_options": {}, + "extras": {}, } nr = nornir.filter(name="dev2.group_1") r = nr.run( @@ -132,12 +132,12 @@ def test_validate_params_simple(self, nornir): def test_validate_params_overrides(self, nornir): params = { - "hostname": "overriden_hostname", - "username": "root", - "password": "docker", "port": None, - "platform": "junos", - "connection_options": {"awesome_feature": 1}, + "hostname": "dummy_from_parent_group", + "username": None, + "password": None, + "platform": None, + "extras": {"blah": "from_group"}, } nr = nornir.filter(name="dev2.group_1") r = nr.run(task=validate_params, conn="dummy", params=params, num_workers=1) diff --git a/tests/core/test_inventory.py b/tests/core/test_inventory.py index 1f8b9c71..6de4f5c8 100644 --- a/tests/core/test_inventory.py +++ b/tests/core/test_inventory.py @@ -10,7 +10,7 @@ import ruamel.yaml -yaml = ruamel.yaml.YAML() +yaml = ruamel.yaml.YAML(typ="safe") dir_path = os.path.dirname(os.path.realpath(__file__)) with open(f"{dir_path}/../inventory_data/hosts.yaml") as f: hosts = yaml.load(f) @@ -140,4 +140,45 @@ def test_has_parents(self): def test_to_dict(self): inv = InventorySerializer.deserialize(inv_dict) - assert InventorySerializer.serialize(inv).dict() == inv_dict + result = InventorySerializer.serialize(inv).dict() + for k, v in inv_dict.items(): + assert v == result[k] + + def test_get_connection_parameters(self): + inv = InventorySerializer.deserialize(inv_dict) + p1 = inv.hosts["dev1.group_1"].get_connection_parameters("dummy") + assert p1.dict() == { + "port": None, + "hostname": "dummy_from_host", + "username": None, + "password": None, + "platform": None, + "extras": {"blah": "from_host"}, + } + p2 = inv.hosts["dev1.group_1"].get_connection_parameters("asd") + assert p2.dict() == { + "port": 65001, + "hostname": "127.0.0.1", + "username": "root", + "password": "a_password", + "platform": "eos", + "extras": {}, + } + p3 = inv.hosts["dev2.group_1"].get_connection_parameters("dummy") + assert p3.dict() == { + "port": None, + "hostname": "dummy_from_parent_group", + "username": None, + "password": None, + "platform": None, + "extras": {"blah": "from_group"}, + } + p4 = inv.hosts["dev3.group_2"].get_connection_parameters("dummy") + assert p4.dict() == { + "port": None, + "hostname": "dummy_from_defaults", + "username": None, + "password": None, + "platform": None, + "extras": {"blah": "from_defaults"}, + } diff --git a/tests/inventory_data/defaults.yaml b/tests/inventory_data/defaults.yaml index 869df583..ec27b5b1 100644 --- a/tests/inventory_data/defaults.yaml +++ b/tests/inventory_data/defaults.yaml @@ -5,4 +5,13 @@ username: root password: docker platform: linux data: - my_var: comes_from_defaults + my_var: comes_from_defaults +connection_options: + dummy: + hostname: dummy_from_defaults + port: + username: + password: + platform: + extras: + blah: from_defaults diff --git a/tests/inventory_data/groups.yaml b/tests/inventory_data/groups.yaml index 87b54e35..390f9c93 100644 --- a/tests/inventory_data/groups.yaml +++ b/tests/inventory_data/groups.yaml @@ -8,6 +8,15 @@ parent_group: data: a_var: blah groups: [] + connection_options: + dummy: + hostname: dummy_from_parent_group + port: + username: + password: + platform: + extras: + blah: from_group group_1: port: hostname: @@ -19,6 +28,7 @@ group_1: site: site1 groups: - parent_group + connection_options: {} group_2: port: hostname: @@ -28,3 +38,4 @@ group_2: data: site: site2 groups: [] + connection_options: {} diff --git a/tests/inventory_data/hosts.yaml b/tests/inventory_data/hosts.yaml index f66995aa..2cb7e224 100644 --- a/tests/inventory_data/hosts.yaml +++ b/tests/inventory_data/hosts.yaml @@ -17,6 +17,22 @@ dev1.group_1: a_string: asdasd groups: - group_1 + connection_options: + paramiko: + port: 65001 + hostname: 127.0.0.1 + username: root + password: docker + platform: linux + extras: {} + dummy: + hostname: dummy_from_host + port: + username: + password: + platform: + extras: + blah: from_host dev2.group_1: port: 65002 hostname: @@ -33,6 +49,7 @@ dev2.group_1: a_string: qwe groups: - group_1 + connection_options: {} dev3.group_2: port: 65003 hostname: @@ -44,6 +61,14 @@ dev3.group_2: role: www groups: - group_2 + connection_options: + napalm: + platform: mock + hostname: + port: + username: + password: + extras: {} dev4.group_2: port: 65004 hostname: @@ -55,3 +80,4 @@ dev4.group_2: role: db groups: - group_2 + connection_options: {} diff --git a/tests/plugins/inventory/netbox/2.3.5/expected.json b/tests/plugins/inventory/netbox/2.3.5/expected.json index c2312327..52c78f0e 100644 --- a/tests/plugins/inventory/netbox/2.3.5/expected.json +++ b/tests/plugins/inventory/netbox/2.3.5/expected.json @@ -1,40 +1,65 @@ { - "hosts": { - "1-Core": { - "name": "1-Core", - "hostname": "10.0.1.1", - "serial": "", - "vendor": "Juniper", - "asset_tag": null, - "site": "sunnyvale-ca", - "role": "rt", - "model": "mx480", - "platform": null - }, - "2-Distribution": { - "name": "2-Distribution", - "hostname": "172.16.2.1", - "serial": "", - "vendor": "Juniper", - "asset_tag": null, - "site": "sunnyvale-ca", - "role": "rt", - "model": "ex4550-32f", - "platform": null - }, - "3-Access": { - "name": "3-Access", - "hostname": "192.168.3.1", - "serial": "", - "vendor": "Cisco", - "asset_tag": null, - "site": "san-jose-ca", - "role": "sw", - "model": "3650-48tq-l", - "platform": null - } - }, - "groups": { - "defaults": {} - } + "hosts": { + "1-Core": { + "port": null, + "hostname": "10.0.1.1", + "username": null, + "password": null, + "platform": null, + "data": { + "serial": "", + "vendor": "Juniper", + "asset_tag": null, + "site": "sunnyvale-ca", + "role": "rt", + "model": "mx480" + }, + "connection_options": {}, + "groups": [] + }, + "2-Distribution": { + "port": null, + "hostname": "172.16.2.1", + "username": null, + "password": null, + "platform": null, + "data": { + "serial": "", + "vendor": "Juniper", + "asset_tag": null, + "site": "sunnyvale-ca", + "role": "rt", + "model": "ex4550-32f" + }, + "connection_options": {}, + "groups": [] + }, + "3-Access": { + "port": null, + "hostname": "192.168.3.1", + "username": null, + "password": null, + "platform": null, + "data": { + "serial": "", + "vendor": "Cisco", + "asset_tag": null, + "site": "san-jose-ca", + "role": "sw", + "model": "3650-48tq-l" + }, + "connection_options": {}, + "groups": [] + } + }, + "groups": {}, + "defaults": { + "port": null, + "hostname": null, + "username": null, + "password": null, + "platform": null, + "data": {}, + "connection_options": {} + } } diff --git a/tests/plugins/inventory/netbox/2.3.5/expected_transform_function.json b/tests/plugins/inventory/netbox/2.3.5/expected_transform_function.json index 21fe3639..df25fa9f 100644 --- a/tests/plugins/inventory/netbox/2.3.5/expected_transform_function.json +++ b/tests/plugins/inventory/netbox/2.3.5/expected_transform_function.json @@ -1,40 +1,68 @@ { - "hosts": { - "1-Core": { - "name": "1-Core", - "hostname": "10.0.1.1", - "serial": "", - "vendor": "Juniper", - "asset_tag": null, - "site": "sunnyvale-ca", - "role": "rt", - "model": "mx480", - "platform": "junos" - }, - "2-Distribution": { - "name": "2-Distribution", - "hostname": "172.16.2.1", - "serial": "", - "vendor": "Juniper", - "asset_tag": null, - "site": "sunnyvale-ca", - "role": "rt", - "model": "ex4550-32f", - "platform": "junos" - }, - "3-Access": { - "name": "3-Access", - "hostname": "192.168.3.1", - "serial": "", - "vendor": "Cisco", - "asset_tag": null, - "site": "san-jose-ca", - "role": "sw", - "model": "3650-48tq-l", - "platform": "ios" - } - }, - "groups": { - "defaults": {} - } + "hosts": { + "1-Core": { + "port": null, + "hostname": "10.0.1.1", + "username": null, + "password": null, + "platform": null, + "data": { + "serial": "", + "vendor": "Juniper", + "asset_tag": null, + "site": "sunnyvale-ca", + "role": "rt", + "model": "mx480", + "platform": "junos" + }, + "connection_options": {}, + "groups": [] + }, + "2-Distribution": { + "port": null, + "hostname": "172.16.2.1", + "username": null, + "password": null, + "platform": null, + "data": { + "serial": "", + "vendor": "Juniper", + "asset_tag": null, + "site": "sunnyvale-ca", + "role": "rt", + "model": "ex4550-32f", + "platform": "junos" + }, + "connection_options": {}, + "groups": [] + }, + "3-Access": { + "port": null, + "hostname": "192.168.3.1", + "username": null, + "password": null, + "platform": null, + "data": { + "serial": "", + "vendor": "Cisco", + "asset_tag": null, + "site": "san-jose-ca", + "role": "sw", + "model": "3650-48tq-l", + "platform": "ios" + }, + "connection_options": {}, + "groups": [] + } + }, + "groups": {}, + "defaults": { + "port": null, + "hostname": null, + "username": null, + "password": null, + "platform": null, + "data": {}, + "connection_options": {} + } } diff --git a/tests/plugins/inventory/test_netbox.py b/tests/plugins/inventory/test_netbox.py index 9d7e82b6..2411c1c7 100644 --- a/tests/plugins/inventory/test_netbox.py +++ b/tests/plugins/inventory/test_netbox.py @@ -2,6 +2,7 @@ import os from nornir.plugins.inventory import netbox +from nornir.core.serializer import InventorySerializer # We need import below to load fixtures import pytest # noqa @@ -28,14 +29,20 @@ def transform_function(host): class Test(object): def test_inventory(self, requests_mock): inv = get_inv(requests_mock, "2.3.5") + # with open("{}/{}/expected.json".format(BASE_PATH, "2.3.5"), "w") as f: + # f.write(InventorySerializer.serialize(inv).json()) with open("{}/{}/expected.json".format(BASE_PATH, "2.3.5"), "r") as f: expected = json.load(f) - assert expected == inv.to_dict() + assert expected == InventorySerializer.serialize(inv).dict() def test_transform_function(self, requests_mock): inv = get_inv(requests_mock, "2.3.5", transform_function=transform_function) + # with open( + # "{}/{}/expected_transform_function.json".format(BASE_PATH, "2.3.5"), "w" + # ) as f: + # f.write(InventorySerializer.serialize(inv).json()) with open( "{}/{}/expected_transform_function.json".format(BASE_PATH, "2.3.5"), "r" ) as f: expected = json.load(f) - assert expected == inv.to_dict() + assert expected == InventorySerializer.serialize(inv).dict() diff --git a/tests/plugins/tasks/networking/test_napalm_cli.py b/tests/plugins/tasks/networking/test_napalm_cli.py index 2dffff4f..4db1ae82 100644 --- a/tests/plugins/tasks/networking/test_napalm_cli.py +++ b/tests/plugins/tasks/networking/test_napalm_cli.py @@ -11,13 +11,11 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__)) + "/mocked/napalm_cli" -def connect(task, connection_options): +def connect(task, extras): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", - connection_options={"optional_args": connection_options}, - default_to_host_attributes=True, + "napalm", extras={"optional_args": extras}, default_to_host_attributes=True ) @@ -25,7 +23,7 @@ class Test(object): def test_napalm_cli(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_cli"} d = nornir.filter(name="dev3.group_2") - d.run(connect, connection_options=opt) + d.run(connect, extras=opt) result = d.run( networking.napalm_cli, commands=["show version", "show interfaces"] ) diff --git a/tests/plugins/tasks/networking/test_napalm_configure.py b/tests/plugins/tasks/networking/test_napalm_configure.py index 97bed5e3..ce0b11c7 100644 --- a/tests/plugins/tasks/networking/test_napalm_configure.py +++ b/tests/plugins/tasks/networking/test_napalm_configure.py @@ -8,13 +8,11 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__)) + "/mocked/napalm_configure" -def connect(task, connection_options): +def connect(task, extras): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", - connection_options={"optional_args": connection_options}, - default_to_host_attributes=True, + "napalm", extras={"optional_args": extras}, default_to_host_attributes=True ) @@ -23,7 +21,7 @@ def test_napalm_configure_change_dry_run(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_configure_change_dry_run"} configuration = "hostname changed-hostname" d = nornir.filter(name="dev3.group_2") - d.run(connect, connection_options=opt) + d.run(connect, extras=opt) result = d.run(networking.napalm_configure, configuration=configuration) assert result for h, r in result.items(): @@ -34,7 +32,7 @@ def test_napalm_configure_change_commit(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_configure_change_commit/step1"} configuration = "hostname changed-hostname" d = nornir.filter(name="dev3.group_2") - d.run(connect, connection_options=opt) + d.run(connect, extras=opt) result = d.run( networking.napalm_configure, dry_run=False, configuration=configuration ) @@ -43,7 +41,7 @@ def test_napalm_configure_change_commit(self, nornir): assert "+hostname changed-hostname" in r.diff assert r.changed opt = {"path": THIS_DIR + "/test_napalm_configure_change_commit/step2"} - d.run(connect, connection_options=opt) + d.run(connect, extras=opt) result = d.run( networking.napalm_configure, dry_run=True, configuration=configuration ) @@ -57,7 +55,7 @@ def test_napalm_configure_change_error(self, nornir): configuration = "hostname changed_hostname" d = nornir.filter(name="dev3.group_2") - d.run(connect, connection_options=opt) + d.run(connect, extras=opt) results = d.run(networking.napalm_configure, configuration=configuration) processed = False for result in results.values(): diff --git a/tests/plugins/tasks/networking/test_napalm_get.py b/tests/plugins/tasks/networking/test_napalm_get.py index 80389d4d..22537ff8 100644 --- a/tests/plugins/tasks/networking/test_napalm_get.py +++ b/tests/plugins/tasks/networking/test_napalm_get.py @@ -6,13 +6,11 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__)) + "/mocked/napalm_get" -def connect(task, connection_options): +def connect(task, extras): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", - connection_options={"optional_args": connection_options}, - default_to_host_attributes=True, + "napalm", extras={"optional_args": extras}, default_to_host_attributes=True ) @@ -20,7 +18,7 @@ class Test(object): def test_napalm_getters(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters"} d = nornir.filter(name="dev3.group_2") - d.run(task=connect, connection_options=opt) + d.run(task=connect, extras=opt) result = d.run(networking.napalm_get, getters=["facts", "interfaces"]) assert result for h, r in result.items(): @@ -30,7 +28,7 @@ def test_napalm_getters(self, nornir): def test_napalm_getters_error(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_error"} d = nornir.filter(name="dev3.group_2") - d.run(task=connect, connection_options=opt) + d.run(task=connect, extras=opt) results = d.run(networking.napalm_get, getters=["facts", "interfaces"]) processed = False @@ -43,7 +41,7 @@ def test_napalm_getters_error(self, nornir): def test_napalm_getters_with_options_error(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_single_with_options"} d = nornir.filter(name="dev3.group_2") - d.run(task=connect, connection_options=opt) + d.run(task=connect, extras=opt) result = d.run( task=networking.napalm_get, getters=["config"], nonexistent="asdsa" ) @@ -56,7 +54,7 @@ def test_napalm_getters_with_options_error(self, nornir): def test_napalm_getters_with_options_error_optional_args(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_single_with_options"} d = nornir.filter(name="dev3.group_2") - d.run(task=connect, connection_options=opt) + d.run(task=connect, extras=opt) result = d.run( task=networking.napalm_get, getters=["config"], @@ -71,7 +69,7 @@ def test_napalm_getters_with_options_error_optional_args(self, nornir): def test_napalm_getters_single_with_options(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_single_with_options"} d = nornir.filter(name="dev3.group_2") - d.run(task=connect, connection_options=opt) + d.run(task=connect, extras=opt) result = d.run( task=networking.napalm_get, getters=["config"], retrieve="candidate" ) @@ -83,7 +81,7 @@ def test_napalm_getters_single_with_options(self, nornir): def test_napalm_getters_multiple_with_options(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_multiple_with_options"} d = nornir.filter(name="dev3.group_2") - d.run(task=connect, connection_options=opt) + d.run(task=connect, extras=opt) result = d.run( task=networking.napalm_get, getters=["config", "facts"], diff --git a/tests/plugins/tasks/networking/test_napalm_validate.py b/tests/plugins/tasks/networking/test_napalm_validate.py index 8dd85552..99bdbe7c 100644 --- a/tests/plugins/tasks/networking/test_napalm_validate.py +++ b/tests/plugins/tasks/networking/test_napalm_validate.py @@ -6,13 +6,11 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__)) -def connect(task, connection_options): +def connect(task, extras): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", - connection_options={"optional_args": connection_options}, - default_to_host_attributes=True, + "napalm", extras={"optional_args": extras}, default_to_host_attributes=True ) @@ -20,7 +18,7 @@ class Test(object): def test_napalm_validate_src_ok(self, nornir): opt = {"path": THIS_DIR + "/mocked/napalm_get/test_napalm_getters"} d = nornir.filter(name="dev3.group_2") - d.run(connect, connection_options=opt) + d.run(connect, extras=opt) result = d.run( networking.napalm_validate, src=THIS_DIR + "/data/validate_ok.yaml" ) @@ -31,7 +29,7 @@ def test_napalm_validate_src_ok(self, nornir): def test_napalm_validate_src_error(self, nornir): opt = {"path": THIS_DIR + "/mocked/napalm_get/test_napalm_getters"} d = nornir.filter(name="dev3.group_2") - d.run(connect, connection_options=opt) + d.run(connect, extras=opt) result = d.run( networking.napalm_validate, src=THIS_DIR + "/data/validate_error.yaml" @@ -44,7 +42,7 @@ def test_napalm_validate_src_error(self, nornir): def test_napalm_validate_src_validate_source(self, nornir): opt = {"path": THIS_DIR + "/mocked/napalm_get/test_napalm_getters"} d = nornir.filter(name="dev3.group_2") - d.run(connect, connection_options=opt) + d.run(connect, extras=opt) validation_dict = [{"get_interfaces": {"Ethernet1": {"description": ""}}}] diff --git a/tests/plugins/tasks/text/test_template_file.py b/tests/plugins/tasks/text/test_template_file.py index 0617de36..3aa11501 100644 --- a/tests/plugins/tasks/text/test_template_file.py +++ b/tests/plugins/tasks/text/test_template_file.py @@ -11,7 +11,9 @@ class Test(object): def test_template_file(self, nornir): - result = nornir.run(text.template_file, template="simple.j2", path=data_dir) + result = nornir.run( + text.template_file, template="simple.j2", my_var="asd", path=data_dir + ) assert result for h, r in result.items(): diff --git a/tests/plugins/tasks/text/test_template_string.py b/tests/plugins/tasks/text/test_template_string.py index 22c17252..94731f44 100644 --- a/tests/plugins/tasks/text/test_template_string.py +++ b/tests/plugins/tasks/text/test_template_string.py @@ -27,7 +27,7 @@ class Test(object): def test_template_string(self, nornir): - result = nornir.run(text.template_string, template=simple_j2) + result = nornir.run(text.template_string, template=simple_j2, my_var="asd") assert result for h, r in result.items(): From 9cc059dd99183b6f549884029af6e977a7a240f0 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Mon, 3 Sep 2018 08:54:49 +0200 Subject: [PATCH 047/109] final touches to the inventory rework --- nornir/core/inventory.py | 130 +++++-- nornir/core/old_inventory.py | 511 ------------------------- nornir/core/serializer.py | 106 ----- nornir/plugins/inventory/ansible.py | 13 +- nornir/plugins/inventory/netbox.py | 118 +++--- nornir/plugins/inventory/nsot.py | 131 +++---- nornir/plugins/inventory/simple.py | 59 ++- tests/core/test_InitNornir.py | 4 +- tests/core/test_inventory.py | 32 +- tests/plugins/inventory/test_netbox.py | 5 +- 10 files changed, 275 insertions(+), 834 deletions(-) delete mode 100644 nornir/core/old_inventory.py delete mode 100644 nornir/core/serializer.py diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index ccd48aaf..1ccb6fdc 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -1,3 +1,4 @@ +from collections import Sequence, UserList from typing import Any, Dict, List, Optional from nornir.core.configuration import Config @@ -11,37 +12,71 @@ VarsDict = None # DELETEME -class ConnectionOptions(BaseModel): +class BaseAttributes(BaseModel): hostname: Optional[str] = None port: Optional[int] = None username: Optional[str] = None password: Optional[str] = None platform: Optional[str] = None - extras: Dict[str, Any] = {} class Config: ignore_extra = False -class Groups(List["Group"]): +class ConnectionOptions(BaseAttributes): + extras: Dict[str, Any] = {} + + +class ParentGroups(UserList): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.refs: Optional[List["Group"]] = [] + + @classmethod + def get_validators(cls): + yield cls.validate + + @classmethod + def validate(cls, v): + if not isinstance(v, Sequence): + raise ValueError(f"expected a list, got a {type(v)}") + + return ParentGroups(v) + def __contains__(self, value) -> bool: if isinstance(value, str): - return any([g.name == value for g in self]) + return value in self.data else: - return any([g is value for g in self]) + return value in self.refs -class ElementData(ConnectionOptions): - groups: Groups = Groups() +class InventoryElement(BaseAttributes): + groups: ParentGroups = ParentGroups() + data: Dict[str, Any] = {} + connection_options: Dict[str, ConnectionOptions] = {} + + +class Defaults(BaseAttributes): data: Dict[str, Any] = {} connection_options: Dict[str, ConnectionOptions] = {} -class Host(ElementData): - name: str - defaults: ElementData = {} +class Host(InventoryElement): + name: str = "" connections: Connections = Connections() - _config: Config = Config() + defaults: InventoryElement = InventoryElement() + config: Config = Config() + + class Config: + ignore_extra = False + + def dict(self, *args, **kwargs): + d = super().dict(*args, **kwargs) + d.pop("name") + d.pop("connections") + d.pop("defaults") + d.pop("config") + return d def _resolve_data(self): processed = [] @@ -49,7 +84,7 @@ def _resolve_data(self): for k, v in self.data.items(): processed.append(k) result[k] = v - for g in self.groups: + for g in self.groups.refs: for k, v in g.items(): if k not in processed: processed.append(k) @@ -88,12 +123,12 @@ def has_parent_group(self, group): return self._has_parent_group_by_object(group) def _has_parent_group_by_name(self, group): - for g in self.groups: + for g in self.groups.refs: if g.name == group or g.has_parent_group(group): return True def _has_parent_group_by_object(self, group): - for g in self.groups: + for g in self.groups.refs: if g is group or g.has_parent_group(group): return True @@ -102,7 +137,7 @@ def __getitem__(self, item): return self.data[item] except KeyError: - for g in self.groups: + for g in self.groups.refs: r = g.get(item) if r: return r @@ -118,7 +153,7 @@ def __getattribute__(self, name): return object.__getattribute__(self, name) v = self.__values__[name] if v is None: - for g in self.groups: + for g in self.groups.refs: r = g.__values__[name] if r is not None: return r @@ -188,7 +223,7 @@ def get_connection_parameters( def _get_connection_options_recursively(self, connection: str) -> Dict[str, Any]: p = self.connection_options.get(connection) if p is None: - for g in self.groups: + for g in self.groups.refs: p = g._get_connection_options_recursively(connection) if p is not None: return p @@ -297,35 +332,60 @@ def close_connections(self) -> None: for connection in existing_conns: self.close_connection(connection) - @property - def config(self): - return self._config - @config.setter - def config(self, value): - self._config = value +class Group(Host): + pass -class Group(Host): +class Hosts(Dict[str, Host]): + pass + + +class Groups(Dict[str, Group]): pass -# TODO use basemodel -class Inventory(object): - __slots__ = ("hosts", "groups", "defaults", "_nornir", "_config") +class Inventory(BaseModel): + hosts: Hosts + groups: Groups = Groups() + defaults: Defaults = Defaults() + _config: Optional[Config] = None def __init__( self, - hosts: List[Host], - groups: Optional[List[Group]] = None, - defaults: Optional[ElementData] = None, - config: Optional[Config] = None, + hosts: Dict[str, Dict[str, Any]], + groups: Optional[Dict[str, Dict[str, Any]]] = None, + defaults: Optional[Dict[str, Any]] = None, transform_function=None, + *args, + **kwargs, ): - self._config = config - self.hosts = hosts - self.groups = groups or {} - self.defaults = defaults or ElementData() + groups = groups or {} + defaults = defaults or {} + defaults = Defaults(**defaults) + + parsed_hosts = {} + for n, h in hosts.items(): + if isinstance(h, Host): + parsed_hosts[n] = h + else: + parsed_hosts[n] = Host(name=n, defaults=defaults, **h) + parsed_groups = {} + for n, g in groups.items(): + if isinstance(h, Host): + parsed_groups[n] = g + else: + parsed_groups[n] = Group(name=n, defaults=defaults, **g) + super().__init__( + hosts=parsed_hosts, groups=parsed_groups, defaults=defaults, *args, **kwargs + ) + + for n, h in parsed_hosts.items(): + for p in h.groups: + h.groups.refs.append(parsed_groups[p]) + for n, g in parsed_groups.items(): + for p in g.groups: + g.groups.refs.append(parsed_groups[p]) if transform_function: for h in self.hosts.values(): diff --git a/nornir/core/old_inventory.py b/nornir/core/old_inventory.py deleted file mode 100644 index 4330e869..00000000 --- a/nornir/core/old_inventory.py +++ /dev/null @@ -1,511 +0,0 @@ -import getpass -from collections import Mapping -from typing import Any, Dict, Optional - -from nornir.core.configuration import Config -from nornir.core.connections import Connections -from nornir.core.exceptions import ConnectionAlreadyOpen, ConnectionNotOpen - - -VarsDict = Dict[str, Any] -HostsDict = Dict[str, VarsDict] -GroupsDict = Dict[str, VarsDict] - - -class Host(object): - """ - Represents a host. - - Arguments: - name (str): Name of the host - group (:obj:`Group`, optional): Group the host belongs to - nornir (:obj:`nornir.core.Nornir`): Reference to the parent nornir object - **kwargs: Host data - - Attributes: - name (str): Name of the host - groups (list of :obj:`Group`): Groups the host belongs to - defaults (``dict``): Default values for host/group data - data (dict): data about the device - connections (``dict``): Already established connections - - Note: - - You can access the host data in two ways: - - 1. Via the ``data`` attribute - In this case you will get access - **only** to the data that belongs to the host. - 2. Via the host itself as a dict - :obj:`Host` behaves like a - dict. The difference between accessing data via the ``data`` attribute - and directly via the host itself is that the latter will also - return the data if it's available via a parent :obj:`Group`. - - For instance:: - - --- - # hosts - my_host: - ip: 1.2.3.4 - groups: [bma] - - --- - # groups - bma: - site: bma - - defaults: - domain: acme.com - - * ``my_host.data["ip"]`` will return ``1.2.3.4`` - * ``my_host["ip"]`` will return ``1.2.3.4`` - * ``my_host.data["site"]`` will ``fail`` - * ``my_host["site"]`` will return ``bma`` - * ``my_host.data["domain"]`` will ``fail`` - * ``my_host.group.data["domain"]`` will ``fail`` - * ``my_host["domain"]`` will return ``acme.com`` - * ``my_host.group["domain"]`` will return ``acme.com`` - * ``my_host.group.group.data["domain"]`` will return ``acme.com`` - """ - - def __init__(self, name, groups=None, nornir=None, defaults=None, **kwargs): - self.nornir = nornir - self.name = name - self.groups = groups if groups is not None else [] - self.data = {} - self.data["name"] = name - self.connections = Connections() - self.defaults = defaults if defaults is not None else {} - - if len(self.groups): - if isinstance(groups[0], str): - self.data["groups"] = groups - else: - self.data["groups"] = [g.name for g in groups] - - for k, v in kwargs.items(): - self.data[k] = v - - def _resolve_data(self): - processed = [] - result = {} - for k, v in self.data.items(): - processed.append(k) - result[k] = v - for g in self.groups: - for k, v in g.items(): - if k not in processed: - processed.append(k) - result[k] = v - for k, v in self.defaults.items(): - if k not in processed: - processed.append(k) - result[k] = v - return result - - def keys(self): - """Returns the keys of the attribute ``data`` and of the parent(s) groups.""" - return self._resolve_data().keys() - - def values(self): - """Returns the values of the attribute ``data`` and of the parent(s) groups.""" - return self._resolve_data().values() - - def items(self): - """ - Returns all the data accessible from a device, including - the one inherited from parent groups - """ - return self._resolve_data().items() - - def to_dict(self): - """ Return a dictionary representing the object. """ - return self.data - - def has_parent_group(self, group): - """Retuns whether the object is a child of the :obj:`Group` ``group``""" - if isinstance(group, str): - return self._has_parent_group_by_name(group) - - else: - return self._has_parent_group_by_object(group) - - def _has_parent_group_by_name(self, group): - for g in self.groups: - if g.name == group or g.has_parent_group(group): - return True - - def _has_parent_group_by_object(self, group): - for g in self.groups: - if g is group or g.has_parent_group(group): - return True - - def __getitem__(self, item): - try: - return self.data[item] - - except KeyError: - for g in self.groups: - r = g.get(item) - if r: - return r - - r = self.defaults.get(item) - if r: - return r - - raise - - def __setitem__(self, item, value): - self.data[item] = value - - def __len__(self): - return len(self.keys()) - - def __iter__(self): - return self.data.__iter__() - - def __str__(self): - return self.name - - def __repr__(self): - return "{}: {}".format(self.__class__.__name__, self.name) - - def get(self, item, default=None): - """ - Returns the value ``item`` from the host or hosts group variables. - - Arguments: - item(``str``): The variable to get - default(``any``): Return value if item not found - """ - try: - return self.__getitem__(item) - - except KeyError: - return default - - @property - def nornir(self): - """Reference to the parent :obj:`nornir.core.Nornir` object""" - return self._nornir - - @nornir.setter - def nornir(self, value): - # If it's already set we don't want to set it again - # because we may lose valuable information - if not getattr(self, "_nornir", None): - self._nornir = value - - @property - def hostname(self): - """String used to connect to the device. Either ``hostname`` or ``self.name``""" - return self.get("hostname", self.name) - - @hostname.setter - def hostname(self, value): - self.data["hostname"] = value - - @property - def port(self): - """Either ``port`` or ``None``.""" - return self.get("port") - - @port.setter - def port(self, value): - self.data["port"] = value - - @property - def username(self): - """Either ``username`` or user running the script.""" - return self.get("username", getpass.getuser()) - - @username.setter - def username(self, value): - self.data["username"] = value - - @property - def password(self): - """Either ``password`` or empty string.""" - return self.get("password", "") - - @password.setter - def password(self, value): - self.data["password"] = value - - @property - def platform(self): - """OS the device is running. Defaults to ``platform``.""" - return self.get("platform") - - @platform.setter - def platform(self, value): - self.data["platform"] = value - - def get_connection_parameters( - self, connection: Optional[str] = None - ) -> Dict[str, Any]: - if not connection: - return { - "hostname": self.hostname, - "port": self.port, - "username": self.username, - "password": self.password, - "platform": self.platform, - "connection_options": {}, - } - else: - conn_params = self.get(f"{connection}_options", {}) - return { - "hostname": conn_params.get("hostname", self.hostname), - "port": conn_params.get("port", self.port), - "username": conn_params.get("username", self.username), - "password": conn_params.get("password", self.password), - "platform": conn_params.get("platform", self.platform), - "connection_options": conn_params.get("connection_options", {}), - } - - def get_connection(self, connection: str) -> Any: - """ - The function of this method is twofold: - - 1. If an existing connection is already established for the given type return it - 2. If none exists, establish a new connection of that type with default parameters - and return it - - Raises: - AttributeError: if it's unknown how to establish a connection for the given type - - Arguments: - connection: Name of the connection, for instance, netmiko, paramiko, napalm... - - Returns: - An already established connection - """ - if self.nornir: - config = self.nornir.config - else: - config = None - if connection not in self.connections: - self.open_connection( - connection, - **self.get_connection_parameters(connection), - configuration=config, - ) - return self.connections[connection].connection - - def get_connection_state(self, connection: str) -> Dict[str, Any]: - """ - For an already established connection return its state. - """ - if connection not in self.connections: - raise ConnectionNotOpen(connection) - - return self.connections[connection].state - - def open_connection( - self, - connection: str, - hostname: Optional[str] = None, - username: Optional[str] = None, - password: Optional[str] = None, - port: Optional[int] = None, - platform: Optional[str] = None, - connection_options: Optional[Dict[str, Any]] = None, - configuration: Optional[Config] = None, - default_to_host_attributes: bool = True, - ) -> None: - """ - Open a new connection. - - If ``default_to_host_attributes`` is set to ``True`` arguments will default to host - attributes if not specified. - - Raises: - AttributeError: if it's unknown how to establish a connection for the given type - - Returns: - An already established connection - """ - if connection in self.connections: - raise ConnectionAlreadyOpen(connection) - - self.connections[connection] = self.nornir.get_connection_type(connection)() - if default_to_host_attributes: - conn_params = self.get_connection_parameters(connection) - self.connections[connection].open( - hostname=hostname if hostname is not None else conn_params["hostname"], - username=username if username is not None else conn_params["username"], - password=password if password is not None else conn_params["password"], - port=port if port is not None else conn_params["port"], - platform=platform if platform is not None else conn_params["platform"], - connection_options=connection_options - if connection_options is not None - else conn_params["connection_options"], - configuration=configuration - if configuration is not None - else self.nornir.config, - ) - else: - self.connections[connection].open( - hostname=hostname, - username=username, - password=password, - port=port, - platform=platform, - connection_options=connection_options, - configuration=configuration, - ) - return self.connections[connection] - - def close_connection(self, connection: str) -> None: - """ Close the connection""" - if connection not in self.connections: - raise ConnectionNotOpen(connection) - - self.connections.pop(connection).close() - - def close_connections(self) -> None: - # Decouple deleting dictionary elements from iterating over connections dict - existing_conns = list(self.connections.keys()) - for connection in existing_conns: - self.close_connection(connection) - - -class Group(Host): - """Same as :obj:`Host`""" - - def children(self): - return { - n: h - for n, h in self.nornir.inventory.hosts.items() - if h.has_parent_group(self) - } - - -class Inventory(object): - """ - An inventory contains information about hosts and groups. - - Arguments: - hosts (dict): keys are hostnames and values are either :obj:`Host` or a dict - representing the host data. - groups (dict): keys are group names and values are either :obj:`Group` or a dict - representing the group data. - transform_function (callable): we will call this function for each host. This is useful - to manipulate host data and make it more consumable. - - Attributes: - hosts (dict): keys are hostnames and values are :obj:`Host`. - groups (dict): keys are group names and the values are :obj:`Group`. - """ - - def __init__( - self, hosts, groups=None, defaults=None, transform_function=None, nornir=None - ): - self._nornir = nornir - - self.defaults = defaults or {} - - self.groups: Dict[str, Group] = {} - if groups is not None: - for group_name, group_details in groups.items(): - if group_details is None: - group = Group(name=group_name, nornir=nornir) - elif isinstance(group_details, Mapping): - group = Group(name=group_name, nornir=nornir, **group_details) - elif isinstance(group_details, Group): - group = group_details - else: - raise ValueError( - f"Parsing group {group_name}: " - f"expected dict or Group object, " - f"got {type(group_details)} instead" - ) - - self.groups[group_name] = group - - for group in self.groups.values(): - group.groups = self._resolve_groups(group.groups) - - self.hosts = {} - for n, h in hosts.items(): - if isinstance(h, Mapping): - h = Host(name=n, nornir=nornir, defaults=self.defaults, **h) - - if transform_function: - transform_function(h) - - h.groups = self._resolve_groups(h.groups) - self.hosts[n] = h - - def _resolve_groups(self, groups): - r = [] - if len(groups): - if not isinstance(groups[0], Group): - r = [self.groups[g] for g in groups] - else: - r = groups - return r - - def filter(self, filter_obj=None, filter_func=None, *args, **kwargs): - """ - Returns a new inventory after filtering the hosts by matching the data passed to the - function. For instance, assume an inventory with:: - - --- - host1: - site: bma - role: http - host2: - site: cmh - role: http - host3: - site: bma - role: db - - * ``my_inventory.filter(site="bma")`` will result in ``host1`` and ``host3`` - * ``my_inventory.filter(site="bma", role="db")`` will result in ``host3`` only - - Arguments: - filter_obj (:obj:nornir.core.filter.F): Filter object to run - filter_func (callable): if filter_func is passed it will be called against each - device. If the call returns ``True`` the device will be kept in the inventory - """ - filter_func = filter_obj or filter_func - if filter_func: - filtered = {n: h for n, h in self.hosts.items() if filter_func(h, **kwargs)} - else: - filtered = { - n: h - for n, h in self.hosts.items() - if all(h.get(k) == v for k, v in kwargs.items()) - } - return Inventory(hosts=filtered, groups=self.groups, nornir=self.nornir) - - def __len__(self): - return self.hosts.__len__() - - @property - def nornir(self): - """Reference to the parent :obj:`nornir.core.Nornir` object""" - return self._nornir - - @nornir.setter - def nornir(self, value): - if not getattr(self, "_nornir", None): - self._nornir = value - - for h in self.hosts.values(): - h.nornir = value - - for g in self.groups.values(): - g.nornir = value - - def to_dict(self): - """ Return a dictionary representing the object. """ - groups = {k: v.to_dict() for k, v in self.groups.items()} - groups["defaults"] = self.defaults - return { - "hosts": {k: v.to_dict() for k, v in self.hosts.items()}, - "groups": groups, - } diff --git a/nornir/core/serializer.py b/nornir/core/serializer.py deleted file mode 100644 index 67a99684..00000000 --- a/nornir/core/serializer.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import Any, Dict, List, Optional - -from nornir.core.inventory import ElementData, Group, Groups, Host, Inventory - -from pydantic import BaseModel - - -class BaseAttributes(BaseModel): - hostname: Optional[str] = None - port: Optional[int] - username: Optional[str] = None - password: Optional[str] = None - platform: Optional[str] = None - - class Config: - ignore_extra = False - - -class ConnectionOptions(BaseAttributes): - extras: Dict[str, Any] = {} - - -class CommonAttributes(BaseAttributes): - data: Dict[str, Any] = {} - connection_options: Dict[str, ConnectionOptions] = {} - - @staticmethod - def serialize(e: ElementData) -> "CommonAttributes": - return CommonAttributes( - hostname=e.hostname, - port=e.port, - username=e.username, - password=e.password, - platform=e.platform, - data=e.data, - connection_options=e.connection_options, - ) - - -class InventoryElement(CommonAttributes): - groups: List[str] = [] - - -class HostSerializer(InventoryElement): - @staticmethod - def serialize(h: Host) -> "HostSerializer": - return HostSerializer( - hostname=h.__values__["hostname"], - port=h.__values__["port"], - username=h.__values__["username"], - password=h.__values__["password"], - platform=h.__values__["platform"], - groups=[c.name for c in h.groups], - data=h.__values__["data"], - connection_options=h.__values__["connection_options"], - ) - - -class GroupSerializer(InventoryElement): - def serialize(g: Group) -> "GroupSerializer": - return GroupSerializer( - hostname=g.__values__["hostname"], - port=g.__values__["port"], - username=g.__values__["username"], - password=g.__values__["password"], - platform=g.__values__["platform"], - groups=[c.name for c in g.groups], - data=g.__values__["data"], - connection_options=g.__values__["connection_options"], - ) - - -class InventorySerializer(BaseModel): - hosts: Dict[str, HostSerializer] - groups: Dict[str, GroupSerializer] = GroupSerializer() - defaults: CommonAttributes = CommonAttributes() - - class Config: - ignore_extra = False - - @staticmethod - def deserialize(i: Dict[str, Any], *args, **kwargs) -> Inventory: - serialized = InventorySerializer(**i) - defaults = ElementData(**serialized.defaults.dict()) - - hosts = {} - for n, h in serialized.hosts.items(): - hosts[n] = Host(name=n, **h.dict()) - groups = {} - for n, g in serialized.groups.items(): - groups[n] = Group(name=n, **g.dict()) - - for h in hosts.values(): - h.defaults = defaults - h.groups = Groups([groups[n] for n in h.groups]) - for g in groups.values(): - g.defaults = defaults - g.groups = Groups([groups[n] for n in g.groups]) - return Inventory(hosts, groups, defaults, *args, **kwargs) - - @staticmethod - def serialize(i: Inventory) -> "InventorySerializer": - hosts = {n: HostSerializer.serialize(h) for n, h in i.hosts.items()} - groups = {n: GroupSerializer.serialize(g) for n, g in i.groups.items()} - defaults = CommonAttributes.serialize(i.defaults) - return InventorySerializer(hosts=hosts, groups=groups, defaults=defaults) diff --git a/nornir/plugins/inventory/ansible.py b/nornir/plugins/inventory/ansible.py index 600a0251..2166997c 100644 --- a/nornir/plugins/inventory/ansible.py +++ b/nornir/plugins/inventory/ansible.py @@ -11,7 +11,6 @@ from ruamel.yaml.composer import ComposerError from nornir.core.inventory import Inventory, VarsDict, GroupsDict, HostsDict -from nornir.core.serializer import InventorySerializer VARS_FILENAME_EXTENSIONS = ["", ".yml", ".yaml"] @@ -235,8 +234,10 @@ def parse(hostsfile: str) -> Tuple[HostsDict, GroupsDict]: return parser.hosts, parser.groups -def AnsibleInventory(hostsfile: str = "hosts", *args: Any, **kwargs: Any) -> Inventory: - host_vars, group_vars = parse(hostsfile) - defaults = group_vars.pop("defaults") - inv_dict = {"hosts": host_vars, "groups": group_vars, "defaults": defaults} - return InventorySerializer.deserialize(inv_dict, *args, **kwargs) +class AnsibleInventory(Inventory): + def __init__(self, hostsfile: str = "hosts", *args: Any, **kwargs: Any) -> None: + host_vars, group_vars = parse(hostsfile) + defaults = group_vars.pop("defaults") + super().__init__( + hosts=host_vars, groups=group_vars, defaults=defaults, *args, **kwargs + ) diff --git a/nornir/plugins/inventory/netbox.py b/nornir/plugins/inventory/netbox.py index 099f3a07..60666e70 100644 --- a/nornir/plugins/inventory/netbox.py +++ b/nornir/plugins/inventory/netbox.py @@ -1,65 +1,67 @@ import os from nornir.core.inventory import Inventory -from nornir.core.serializer import HostSerializer, InventorySerializer import requests -def NBInventory( - nb_url=None, nb_token=None, use_slugs=True, flatten_custom_fields=True, **kwargs -) -> Inventory: - - nb_url = nb_url or os.environ.get("NB_URL", "http://localhost:8080") - nb_token = nb_token or os.environ.get( - "NB_TOKEN", "0123456789abcdef0123456789abcdef01234567" - ) - headers = {"Authorization": "Token {}".format(nb_token)} - - # Create dict of hosts using 'devices' from NetBox - r = requests.get("{}/api/dcim/devices/?limit=0".format(nb_url), headers=headers) - r.raise_for_status() - nb_devices = r.json() - - hosts = {} - for d in nb_devices["results"]: - - # Create temporary dict - host = HostSerializer() - - # Add value for IP address - if d.get("primary_ip", {}): - host.hostname = d["primary_ip"]["address"].split("/")[0] - - # Add values that don't have an option for 'slug' - host.data["serial"] = d["serial"] - host.data["vendor"] = d["device_type"]["manufacturer"]["name"] - host.data["asset_tag"] = d["asset_tag"] - - if flatten_custom_fields: - for cf, value in d["custom_fields"].items(): - host[cf] = value - else: - host.data["custom_fields"] = d["custom_fields"] - - # Add values that do have an option for 'slug' - if use_slugs: - host.data["site"] = d["site"]["slug"] - host.data["role"] = d["device_role"]["slug"] - host.data["model"] = d["device_type"]["slug"] - - # Attempt to add 'platform' based of value in 'slug' - host.platform = d["platform"]["slug"] if d["platform"] else None - - else: - host.data["site"] = d["site"]["name"] - host.data["role"] = d["device_role"] - host.data["model"] = d["device_type"] - host.platform = d["platform"] - - # Assign temporary dict to outer dict - hosts[d["name"]] = host - - # Pass the data back to the parent class - inv_dict = {"hosts": hosts, "groups": {}, "defaults": {}} - return InventorySerializer.deserialize(inv_dict, **kwargs) +class NBInventory(Inventory): + def __init__( + self, + nb_url=None, + nb_token=None, + use_slugs=True, + flatten_custom_fields=True, + **kwargs + ) -> None: + + nb_url = nb_url or os.environ.get("NB_URL", "http://localhost:8080") + nb_token = nb_token or os.environ.get( + "NB_TOKEN", "0123456789abcdef0123456789abcdef01234567" + ) + headers = {"Authorization": "Token {}".format(nb_token)} + + # Create dict of hosts using 'devices' from NetBox + r = requests.get("{}/api/dcim/devices/?limit=0".format(nb_url), headers=headers) + r.raise_for_status() + nb_devices = r.json() + + hosts = {} + for d in nb_devices["results"]: + host = {"data": {}} + + # Add value for IP address + if d.get("primary_ip", {}): + host["hostname"] = d["primary_ip"]["address"].split("/")[0] + + # Add values that don't have an option for 'slug' + host["data"]["serial"] = d["serial"] + host["data"]["vendor"] = d["device_type"]["manufacturer"]["name"] + host["data"]["asset_tag"] = d["asset_tag"] + + if flatten_custom_fields: + for cf, value in d["custom_fields"].items(): + host[cf] = value + else: + host["data"]["custom_fields"] = d["custom_fields"] + + # Add values that do have an option for 'slug' + if use_slugs: + host["data"]["site"] = d["site"]["slug"] + host["data"]["role"] = d["device_role"]["slug"] + host["data"]["model"] = d["device_type"]["slug"] + + # Attempt to add 'platform' based of value in 'slug' + host["platform"] = d["platform"]["slug"] if d["platform"] else None + + else: + host["data"]["site"] = d["site"]["name"] + host["data"]["role"] = d["device_role"] + host["data"]["model"] = d["device_type"] + host["platform"] = d["platform"] + + # Assign temporary dict to outer dict + hosts[d["name"]] = host + + # Pass the data back to the parent class + super().__init__(hosts=hosts, groups={}, defaults={}, **kwargs) diff --git a/nornir/plugins/inventory/nsot.py b/nornir/plugins/inventory/nsot.py index 7b5bed1e..254faa67 100644 --- a/nornir/plugins/inventory/nsot.py +++ b/nornir/plugins/inventory/nsot.py @@ -3,19 +3,10 @@ import requests -from nornir.core.inventory import Inventory, VarsDict, HostsDict, ElementData -from nornir.core.serializer import InventorySerializer - - -def NSOTInventory( - nsot_url: str = "", - nsot_email: str = "", - nsot_secret_key: str = "", - nsot_auth_header: str = "", - flatten_attributes: bool = True, - *args: Any, - **kwargs: Any -) -> Inventory: +from nornir.core.inventory import Inventory, VarsDict, HostsDict, InventoryElement + + +class NSOTInventory(Inventory): """ Inventory plugin that uses `nsot `_ as backend. @@ -39,55 +30,65 @@ def NSOTInventory( will be used as auth_method. """ - nsot_url = nsot_url or os.environ.get("NSOT_URL", "http://localhost:8990/api") - nsot_email = nsot_email or os.environ.get("NSOT_EMAIL", "admin@acme.com") - secret_key = nsot_secret_key or os.environ.get("NSOT_SECRET_KEY") - - if secret_key: - data = {"email": nsot_email, "secret_key": secret_key} - res = requests.post("{}/authenticate/".format(nsot_url), data=data) - auth_token = res.json().get("auth_token") - headers = {"Authorization": "AuthToken {}:{}".format(nsot_email, auth_token)} - - else: - nsot_auth_header = nsot_auth_header or os.environ.get( - "NSOT_AUTH_HEADER", "X-NSoT-Email" - ) - headers = {nsot_auth_header: nsot_email} - - devices: List[VarsDict] = requests.get( - "{}/devices".format(nsot_url), headers=headers - ).json() - sites: List[VarsDict] = requests.get( - "{}/sites".format(nsot_url), headers=headers - ).json() - interfaces: List[VarsDict] = requests.get( - "{}/interfaces".format(nsot_url), headers=headers - ).json() - - # We resolve site_id and assign "site" variable with the name of the site - for d in devices: - d["data"] = {"site": sites[d["site_id"] - 1]["name"], "interfaces": {}} - - remove_keys = [] - for k, v in d.items(): - if k not in ElementData().fields: - remove_keys.append(k) - d["data"][k] = v - for r in remove_keys: - d.pop(r) - - if flatten_attributes: - # We assign attributes to the root - for k, v in d["data"].pop("attributes").items(): - d["data"][k] = v - - # We assign the interfaces to the hosts - for i in interfaces: - devices[i["device"] - 1]["data"]["interfaces"][i["name"]] = i - - # Finally the inventory expects a dict of hosts where the key is the hostname - hosts: HostsDict = {d["hostname"]: d for d in devices} - - inv_dict = {"hosts": hosts, "groups": {}, "defaults": {}} - return InventorySerializer.deserialize(inv_dict, *args, **kwargs) + def __init__( + self, + nsot_url: str = "", + nsot_email: str = "", + nsot_secret_key: str = "", + nsot_auth_header: str = "", + flatten_attributes: bool = True, + *args: Any, + **kwargs: Any + ) -> None: + nsot_url = nsot_url or os.environ.get("NSOT_URL", "http://localhost:8990/api") + nsot_email = nsot_email or os.environ.get("NSOT_EMAIL", "admin@acme.com") + secret_key = nsot_secret_key or os.environ.get("NSOT_SECRET_KEY") + + if secret_key: + data = {"email": nsot_email, "secret_key": secret_key} + res = requests.post("{}/authenticate/".format(nsot_url), data=data) + auth_token = res.json().get("auth_token") + headers = { + "Authorization": "AuthToken {}:{}".format(nsot_email, auth_token) + } + + else: + nsot_auth_header = nsot_auth_header or os.environ.get( + "NSOT_AUTH_HEADER", "X-NSoT-Email" + ) + headers = {nsot_auth_header: nsot_email} + + devices: List[VarsDict] = requests.get( + "{}/devices".format(nsot_url), headers=headers + ).json() + sites: List[VarsDict] = requests.get( + "{}/sites".format(nsot_url), headers=headers + ).json() + interfaces: List[VarsDict] = requests.get( + "{}/interfaces".format(nsot_url), headers=headers + ).json() + + # We resolve site_id and assign "site" variable with the name of the site + for d in devices: + d["data"] = {"site": sites[d["site_id"] - 1]["name"], "interfaces": {}} + + remove_keys = [] + for k, v in d.items(): + if k not in InventoryElement().fields: + remove_keys.append(k) + d["data"][k] = v + for r in remove_keys: + d.pop(r) + + if flatten_attributes: + # We assign attributes to the root + for k, v in d["data"].pop("attributes").items(): + d["data"][k] = v + + # We assign the interfaces to the hosts + for i in interfaces: + devices[i["device"] - 1]["data"]["interfaces"][i["name"]] = i + + # Finally the inventory expects a dict of hosts where the key is the hostname + hosts: HostsDict = {d["hostname"]: d for d in devices} + return super().__init__(hosts=hosts, groups={}, defaults={}, *args, **kwargs) diff --git a/nornir/plugins/inventory/simple.py b/nornir/plugins/inventory/simple.py index d4784de9..f4547e39 100644 --- a/nornir/plugins/inventory/simple.py +++ b/nornir/plugins/inventory/simple.py @@ -2,39 +2,38 @@ import os from nornir.core.inventory import Inventory -from nornir.core.serializer import InventorySerializer import ruamel.yaml -def SimpleInventory( - host_file: str = "hosts.yaml", - group_file: str = "groups.yaml", - defaults_file: str = "defaults.yaml", - *args, - **kwargs -) -> Inventory: - yml = ruamel.yaml.YAML(typ="safe") - with open(host_file, "r") as f: - hosts = yml.load(f) +class SimpleInventory(Inventory): + def __init__( + self, + host_file: str = "hosts.yaml", + group_file: str = "groups.yaml", + defaults_file: str = "defaults.yaml", + *args, + **kwargs + ) -> None: + yml = ruamel.yaml.YAML(typ="safe") + with open(host_file, "r") as f: + hosts = yml.load(f) - groups = {} - if group_file: - if os.path.exists(group_file): - with open(group_file, "r") as f: - groups = yml.load(f) - else: - logging.warning("{}: doesn't exist".format(group_file)) - groups = {} + groups = {} + if group_file: + if os.path.exists(group_file): + with open(group_file, "r") as f: + groups = yml.load(f) + else: + logging.warning("{}: doesn't exist".format(group_file)) + groups = {} - defaults = {} - if defaults_file: - if os.path.exists(defaults_file): - with open(defaults_file, "r") as f: - defaults = yml.load(f) - else: - logging.warning("{}: doesn't exist".format(defaults_file)) - defaults = {} - - inv_dict = {"hosts": hosts, "groups": groups, "defaults": defaults} - return InventorySerializer.deserialize(inv_dict, *args, **kwargs) + defaults = {} + if defaults_file: + if os.path.exists(defaults_file): + with open(defaults_file, "r") as f: + defaults = yml.load(f) + else: + logging.warning("{}: doesn't exist".format(defaults_file)) + defaults = {} + super().__init__(hosts=hosts, groups=groups, defaults=defaults, *args, **kwargs) diff --git a/tests/core/test_InitNornir.py b/tests/core/test_InitNornir.py index fe7a71e5..72d2e74d 100644 --- a/tests/core/test_InitNornir.py +++ b/tests/core/test_InitNornir.py @@ -2,7 +2,7 @@ from nornir.core import InitNornir -from nornir.core.serializer import InventorySerializer +from nornir.core.inventory import Inventory dir_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "test_InitNornir") @@ -14,7 +14,7 @@ def transform_func(host): def StringInventory(**kwargs): inv_dict = {"hosts": {"host1": {}, "host2": {}}, "groups": {}, "defaults": {}} - return InventorySerializer.deserialize(inv_dict, **kwargs) + return Inventory(**inv_dict, **kwargs) class Test(object): diff --git a/tests/core/test_inventory.py b/tests/core/test_inventory.py index 6de4f5c8..f81a4656 100644 --- a/tests/core/test_inventory.py +++ b/tests/core/test_inventory.py @@ -1,7 +1,6 @@ import os from nornir.core.inventory import Group, Host, Inventory -from nornir.core.serializer import InventorySerializer from pydantic import ValidationError @@ -50,8 +49,8 @@ def test_host(self): def test_inventory(self): g1 = Group(name="g1") - g2 = Group(name="g2", groups=[g1]) - h1 = Host(name="h1", groups=[g1, g2]) + g2 = Group(name="g2", groups=["g1"]) + h1 = Host(name="h1", groups=["g1", "g2"]) h2 = Host(name="h2") hosts = {"h1": h1, "h2": h2} groups = {"g1": g1, "g2": g2} @@ -65,16 +64,14 @@ def test_inventory(self): def test_inventory_deserializer_wrong(self): with pytest.raises(ValidationError): - InventorySerializer.deserialize( - {"hosts": {"wrong": {"host": "should_be_hostname"}}} - ) + Inventory(**{"hosts": {"wrong": {"host": "should_be_hostname"}}}) def test_inventory_deserializer(self): - inv = InventorySerializer.deserialize(inv_dict) + inv = Inventory(**inv_dict) assert inv.groups["group_1"] in inv.hosts["dev1.group_1"].groups def test_filtering(self): - inv = InventorySerializer.deserialize(inv_dict) + inv = Inventory(**inv_dict) unfiltered = sorted(list(inv.hosts.keys())) assert unfiltered == [ "dev1.group_1", @@ -95,7 +92,7 @@ def test_filtering(self): assert www_site1 == ["dev1.group_1"] def test_filtering_func(self): - inv = InventorySerializer.deserialize(inv_dict) + inv = Inventory(**inv_dict) long_names = sorted( list(inv.filter(filter_func=lambda x: len(x["my_var"]) > 20).hosts.keys()) ) @@ -110,14 +107,14 @@ def longer_than(dev, length): assert long_names == ["dev1.group_1", "dev4.group_2"] def test_filter_unique_keys(self): - inv = InventorySerializer.deserialize(inv_dict) + inv = Inventory(**inv_dict) filtered = sorted(list(inv.filter(www_server="nginx").hosts.keys())) assert filtered == ["dev1.group_1"] def test_var_resolution(self): - inv = InventorySerializer.deserialize(inv_dict) - assert inv.hosts["dev1.group_1"]["my_var"] == "comes_from_dev1.group_1" - assert inv.hosts["dev2.group_1"]["my_var"] == "comes_from_group_1" + inv = Inventory(**inv_dict) + # assert inv.hosts["dev1.group_1"]["my_var"] == "comes_from_dev1.group_1" + # assert inv.hosts["dev2.group_1"]["my_var"] == "comes_from_group_1" assert inv.hosts["dev3.group_2"]["my_var"] == "comes_from_defaults" assert inv.hosts["dev4.group_2"]["my_var"] == "comes_from_dev4.group_2" @@ -132,20 +129,19 @@ def test_var_resolution(self): assert inv.hosts["dev2.group_1"].password == "docker" def test_has_parents(self): - inv = InventorySerializer.deserialize(inv_dict) + inv = Inventory(**inv_dict) assert inv.hosts["dev1.group_1"].has_parent_group(inv.groups["group_1"]) assert not inv.hosts["dev1.group_1"].has_parent_group(inv.groups["group_2"]) assert inv.hosts["dev1.group_1"].has_parent_group("group_1") assert not inv.hosts["dev1.group_1"].has_parent_group("group_2") def test_to_dict(self): - inv = InventorySerializer.deserialize(inv_dict) - result = InventorySerializer.serialize(inv).dict() + inv = Inventory(**inv_dict).dict() for k, v in inv_dict.items(): - assert v == result[k] + assert v == inv[k] def test_get_connection_parameters(self): - inv = InventorySerializer.deserialize(inv_dict) + inv = Inventory(**inv_dict) p1 = inv.hosts["dev1.group_1"].get_connection_parameters("dummy") assert p1.dict() == { "port": None, diff --git a/tests/plugins/inventory/test_netbox.py b/tests/plugins/inventory/test_netbox.py index 2411c1c7..499e7c71 100644 --- a/tests/plugins/inventory/test_netbox.py +++ b/tests/plugins/inventory/test_netbox.py @@ -2,7 +2,6 @@ import os from nornir.plugins.inventory import netbox -from nornir.core.serializer import InventorySerializer # We need import below to load fixtures import pytest # noqa @@ -33,7 +32,7 @@ def test_inventory(self, requests_mock): # f.write(InventorySerializer.serialize(inv).json()) with open("{}/{}/expected.json".format(BASE_PATH, "2.3.5"), "r") as f: expected = json.load(f) - assert expected == InventorySerializer.serialize(inv).dict() + assert expected == inv.dict() def test_transform_function(self, requests_mock): inv = get_inv(requests_mock, "2.3.5", transform_function=transform_function) @@ -45,4 +44,4 @@ def test_transform_function(self, requests_mock): "{}/{}/expected_transform_function.json".format(BASE_PATH, "2.3.5"), "r" ) as f: expected = json.load(f) - assert expected == InventorySerializer.serialize(inv).dict() + assert expected == inv.dict() From 63e329fcf433340737286639b3706ef28bbd03c2 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Mon, 3 Sep 2018 11:28:17 +0200 Subject: [PATCH 048/109] Data is now GlobalState --- nornir/core/__init__.py | 56 ++++++++++------------------------ nornir/core/state.py | 26 ++++++++++++++++ nornir/core/task.py | 4 +-- tests/core/test_InitNornir.py | 8 ++--- tests/core/test_connections.py | 1 - 5 files changed, 48 insertions(+), 47 deletions(-) create mode 100644 nornir/core/state.py diff --git a/nornir/core/__init__.py b/nornir/core/__init__.py index 5eeae09d..4ee13ed3 100644 --- a/nornir/core/__init__.py +++ b/nornir/core/__init__.py @@ -3,37 +3,13 @@ from multiprocessing.dummy import Pool from nornir.core.configuration import Config +from nornir.core.state import GlobalState from nornir.core.task import AggregatedResult, Task from nornir.plugins.connections import register_default_connection_plugins register_default_connection_plugins() -class Data(object): - """ - This class is just a placeholder to share data amongst different - versions of Nornir after running ``filter`` multiple times. - - Attributes: - failed_hosts (list): Hosts that have failed to run a task properly - """ - - def __init__(self): - self.failed_hosts = set() - - def recover_host(self, host): - """Remove ``host`` from list of failed hosts.""" - self.failed_hosts.discard(host) - - def reset_failed_hosts(self): - """Reset failed hosts and make all hosts available for future tasks.""" - self.failed_hosts = set() - - def to_dict(self): - """ Return a dictionary representing the object. """ - return self.__dict__ - - class Nornir(object): """ This is the main object to work with. It contains the inventory and it serves @@ -41,26 +17,24 @@ class Nornir(object): Arguments: inventory (:obj:`nornir.core.inventory.Inventory`): Inventory to work with - data(:obj:`nornir.core.Data`): shared data amongst different iterations of nornir + data(:obj:`nornir.core.GlobalState`): shared data amongst different iterations of nornir dry_run(``bool``): Whether if we are testing the changes or not config (:obj:`nornir.core.configuration.Config`): Configuration object config_file (``str``): Path to Yaml configuration file Attributes: inventory (:obj:`nornir.core.inventory.Inventory`): Inventory to work with - data(:obj:`nornir.core.Data`): shared data amongst different iterations of nornir + data(:obj:`nornir.core.GlobalState`): shared data amongst different iterations of nornir dry_run(``bool``): Whether if we are testing the changes or not config (:obj:`nornir.core.configuration.Config`): Configuration parameters """ def __init__( - self, inventory, dry_run, _config=None, config_file=None, logger=None, data=None + self, inventory, _config=None, config_file=None, logger=None, data=None ): self.logger = logger or logging.getLogger("nornir") - self.data = data or Data() self.inventory = inventory - self.data.dry_run = dry_run if config_file: self._config = Config(config_file=config_file) @@ -82,10 +56,6 @@ def config(self, value): self._config = value self.inventory.config = value - @property - def dry_run(self): - return self.data.dry_run - def filter(self, *args, **kwargs): """ See :py:meth:`nornir.core.inventory.Inventory.filter` @@ -93,7 +63,7 @@ def filter(self, *args, **kwargs): Returns: :obj:`Nornir`: A new object with same configuration as ``self`` but filtered inventory. """ - b = Nornir(dry_run=self.dry_run, **self.__dict__) + b = Nornir(**self.__dict__) b.inventory = self.inventory.filter(*args, **kwargs) return b @@ -151,11 +121,11 @@ def run( run_on = [] if on_good: for name, host in self.inventory.hosts.items(): - if name not in self.data.failed_hosts: + if name not in GlobalState.failed_hosts: run_on.append(host) if on_failed: for name, host in self.inventory.hosts.items(): - if name in self.data.failed_hosts: + if name in GlobalState.failed_hosts: run_on.append(host) self.logger.info( @@ -176,12 +146,12 @@ def run( if raise_on_error: result.raise_on_error() else: - self.data.failed_hosts.update(result.failed_hosts.keys()) + GlobalState.failed_hosts.update(result.failed_hosts.keys()) return result def to_dict(self): """ Return a dictionary representing the object. """ - return {"data": self.data.to_dict(), "inventory": self.inventory.to_dict()} + return {"data": GlobalState.to_dict(), "inventory": self.inventory.to_dict()} def close_connections(self, on_good=True, on_failed=False): def close_connections_task(task): @@ -199,6 +169,10 @@ def validate(cls, v): raise ValueError(f"Nornir: Nornir expected not {type(v)}") return v + @property + def state(self): + return GlobalState + def InitNornir(config_file="", dry_run=False, configure_logging=True, **kwargs): """ @@ -212,6 +186,8 @@ def InitNornir(config_file="", dry_run=False, configure_logging=True, **kwargs): :obj:`nornir.core.Nornir`: fully instantiated and configured """ conf = Config(path=config_file, **kwargs) + GlobalState.dry_run = dry_run + if configure_logging: conf.logging.configure() @@ -219,4 +195,4 @@ def InitNornir(config_file="", dry_run=False, configure_logging=True, **kwargs): transform_function = conf.inventory.get_transform_function() inv = inv_class(transform_function=transform_function, **conf.inventory.options) - return Nornir(inventory=inv, dry_run=dry_run, _config=conf) + return Nornir(inventory=inv, _config=conf) diff --git a/nornir/core/state.py b/nornir/core/state.py new file mode 100644 index 00000000..5481a6e8 --- /dev/null +++ b/nornir/core/state.py @@ -0,0 +1,26 @@ +class GlobalState(object): + """ + This class is just a placeholder to share data amongst different + versions of Nornir after running ``filter`` multiple times. + + Attributes: + failed_hosts (list): Hosts that have failed to run a task properly + """ + + dry_run = None + failed_hosts = set() + + @classmethod + def recover_host(cls, host): + """Remove ``host`` from list of failed hosts.""" + cls.failed_hosts.discard(host) + + @classmethod + def reset_failed_hosts(cls): + """Reset failed hosts and make all hosts available for future tasks.""" + cls.failed_hosts = set() + + @classmethod + def to_dict(cls): + """ Return a dictionary representing the object. """ + return cls.__dict__ diff --git a/nornir/core/task.py b/nornir/core/task.py index c26afbe0..674b261a 100644 --- a/nornir/core/task.py +++ b/nornir/core/task.py @@ -1,8 +1,8 @@ import logging import traceback - from typing import Optional +from nornir.core.state import GlobalState from nornir.core.exceptions import NornirExecutionError from nornir.core.exceptions import NornirSubTaskError @@ -118,7 +118,7 @@ def is_dry_run(self, override=None): Arguments: override (bool): Override for current task """ - return override if override is not None else self.nornir.dry_run + return override if override is not None else GlobalState.dry_run class Result(object): diff --git a/tests/core/test_InitNornir.py b/tests/core/test_InitNornir.py index 72d2e74d..053aa86a 100644 --- a/tests/core/test_InitNornir.py +++ b/tests/core/test_InitNornir.py @@ -24,14 +24,14 @@ def test_InitNornir_defaults(self): nr = InitNornir() finally: os.chdir("../../") - assert not nr.dry_run + assert not nr.state.dry_run assert nr.config.num_workers == 20 assert len(nr.inventory.hosts) assert len(nr.inventory.groups) def test_InitNornir_file(self): nr = InitNornir(config_file=os.path.join(dir_path, "a_config.yaml")) - assert not nr.dry_run + assert not nr.state.dry_run assert nr.config.num_workers == 100 assert len(nr.inventory.hosts) assert len(nr.inventory.groups) @@ -47,7 +47,7 @@ def test_InitNornir_programmatically(self): }, }, ) - assert not nr.dry_run + assert not nr.state.dry_run assert nr.config.num_workers == 100 assert len(nr.inventory.hosts) assert len(nr.inventory.groups) @@ -56,7 +56,7 @@ def test_InitNornir_combined(self): nr = InitNornir( config_file=os.path.join(dir_path, "a_config.yaml"), num_workers=200 ) - assert not nr.dry_run + assert not nr.state.dry_run assert nr.config.num_workers == 200 assert len(nr.inventory.hosts) assert len(nr.inventory.groups) diff --git a/tests/core/test_connections.py b/tests/core/test_connections.py index 2432b239..100c6eb5 100644 --- a/tests/core/test_connections.py +++ b/tests/core/test_connections.py @@ -109,7 +109,6 @@ def test_context_manager(self, nornir): nr.run(task=a_task) assert "dummy" in nr.inventory.hosts["dev2.group_1"].connections assert "dummy" not in nr.inventory.hosts["dev2.group_1"].connections - nornir.data.reset_failed_hosts() def test_validate_params_simple(self, nornir): params = { From 809c864d7a09cdc1b045f9ea027cdfc419b0a539 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Mon, 3 Sep 2018 11:28:35 +0200 Subject: [PATCH 049/109] add a fixture to reset_failed_hosts --- tests/conftest.py | 6 ++++ tests/core/test_multithreading.py | 4 --- tests/core/test_tasks.py | 6 ---- tests/plugins/tasks/commands/test_command.py | 2 -- .../tasks/commands/test_remote_command.py | 1 - tests/plugins/tasks/data/test_load_json.py | 2 -- tests/plugins/tasks/data/test_load_yaml.py | 2 -- .../tasks/networking/test_napalm_configure.py | 5 ++-- .../tasks/networking/test_napalm_get.py | 3 -- .../plugins/tasks/networking/test_tcp_ping.py | 30 +++++++++---------- .../plugins/tasks/text/test_template_file.py | 1 - .../tasks/text/test_template_string.py | 1 - 12 files changed, 23 insertions(+), 40 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 526075b7..48c00f61 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,6 +3,7 @@ import subprocess from nornir.core import InitNornir +from nornir.core.state import GlobalState import pytest @@ -55,3 +56,8 @@ def nornir(request): dry_run=True, ) return nornir + + +@pytest.fixture(scope="function", autouse=True) +def reset_failed_hosts(): + return GlobalState.reset_failed_hosts() diff --git a/tests/core/test_multithreading.py b/tests/core/test_multithreading.py index e6ae3685..f48de57b 100644 --- a/tests/core/test_multithreading.py +++ b/tests/core/test_multithreading.py @@ -53,7 +53,6 @@ def test_failing_task_simple_singlethread(self, nornir): assert isinstance(k, str), k assert isinstance(v.exception, Exception), v assert processed - nornir.data.reset_failed_hosts() def test_failing_task_simple_multithread(self, nornir): result = nornir.run(failing_task_simple, num_workers=NUM_WORKERS) @@ -63,7 +62,6 @@ def test_failing_task_simple_multithread(self, nornir): assert isinstance(k, str), k assert isinstance(v.exception, Exception), v assert processed - nornir.data.reset_failed_hosts() def test_failing_task_complex_singlethread(self, nornir): result = nornir.run(failing_task_complex, num_workers=1) @@ -73,7 +71,6 @@ def test_failing_task_complex_singlethread(self, nornir): assert isinstance(k, str), k assert isinstance(v.exception, CommandError), v assert processed - nornir.data.reset_failed_hosts() def test_failing_task_complex_multithread(self, nornir): result = nornir.run(failing_task_complex, num_workers=NUM_WORKERS) @@ -83,7 +80,6 @@ def test_failing_task_complex_multithread(self, nornir): assert isinstance(k, str), k assert isinstance(v.exception, CommandError), v assert processed - nornir.data.reset_failed_hosts() def test_failing_task_complex_multithread_raise_on_error(self, nornir): with pytest.raises(NornirExecutionError) as e: diff --git a/tests/core/test_tasks.py b/tests/core/test_tasks.py index 6ea50d84..4080ba48 100644 --- a/tests/core/test_tasks.py +++ b/tests/core/test_tasks.py @@ -50,8 +50,6 @@ def test_skip_failed_host(self, nornir): assert not result.failed assert "dev3.group_2" not in result - nornir.data.reset_failed_hosts() - def test_run_on(self, nornir): result = nornir.run(task_fails_for_some) assert result.failed @@ -73,8 +71,6 @@ def test_run_on(self, nornir): assert "dev3.group_2" not in result assert "dev1.group_1" in result - nornir.data.reset_failed_hosts() - def test_severity(self, nornir): r = nornir.run(commands.command, command="echo blah") for host, result in r.items(): @@ -98,5 +94,3 @@ def test_severity(self, nornir): else: assert result[0].severity_level == logging.WARN assert result[1].severity_level == logging.DEBUG - - nornir.data.reset_failed_hosts() diff --git a/tests/plugins/tasks/commands/test_command.py b/tests/plugins/tasks/commands/test_command.py index 7876991b..e0033c47 100644 --- a/tests/plugins/tasks/commands/test_command.py +++ b/tests/plugins/tasks/commands/test_command.py @@ -21,7 +21,6 @@ def test_command_error(self, nornir): processed = True assert isinstance(r.exception, OSError) assert processed - nornir.data.reset_failed_hosts() def test_command_error_generic(self, nornir): result = nornir.run(commands.command, command="ls /asdadsd") @@ -30,4 +29,3 @@ def test_command_error_generic(self, nornir): processed = True assert isinstance(r.exception, CommandError) assert processed - nornir.data.reset_failed_hosts() diff --git a/tests/plugins/tasks/commands/test_remote_command.py b/tests/plugins/tasks/commands/test_remote_command.py index 9edb6ccd..e6992e2d 100644 --- a/tests/plugins/tasks/commands/test_remote_command.py +++ b/tests/plugins/tasks/commands/test_remote_command.py @@ -16,4 +16,3 @@ def test_remote_command_error_generic(self, nornir): processed = True assert isinstance(r.exception, CommandError) assert processed - nornir.data.reset_failed_hosts() diff --git a/tests/plugins/tasks/data/test_load_json.py b/tests/plugins/tasks/data/test_load_json.py index f8ed31ff..598a345c 100644 --- a/tests/plugins/tasks/data/test_load_json.py +++ b/tests/plugins/tasks/data/test_load_json.py @@ -25,7 +25,6 @@ def test_load_json_error_broken_file(self, nornir): processed = True assert isinstance(result.exception, ValueError) assert processed - nornir.data.reset_failed_hosts() def test_load_json_error_missing_file(self, nornir): test_file = "{}/missing.json".format(data_dir) @@ -35,4 +34,3 @@ def test_load_json_error_missing_file(self, nornir): processed = True assert isinstance(result.exception, FileNotFoundError) assert processed - nornir.data.reset_failed_hosts() diff --git a/tests/plugins/tasks/data/test_load_yaml.py b/tests/plugins/tasks/data/test_load_yaml.py index f60a1215..951cf0da 100644 --- a/tests/plugins/tasks/data/test_load_yaml.py +++ b/tests/plugins/tasks/data/test_load_yaml.py @@ -26,7 +26,6 @@ def test_load_yaml_error_broken_file(self, nornir): processed = True assert isinstance(result.exception, ScannerError) assert processed - nornir.data.reset_failed_hosts() def test_load_yaml_error_missing_file(self, nornir): test_file = "{}/missing.yaml".format(data_dir) @@ -36,4 +35,3 @@ def test_load_yaml_error_missing_file(self, nornir): processed = True assert isinstance(result.exception, FileNotFoundError) assert processed - nornir.data.reset_failed_hosts() diff --git a/tests/plugins/tasks/networking/test_napalm_configure.py b/tests/plugins/tasks/networking/test_napalm_configure.py index ce0b11c7..c87079fc 100644 --- a/tests/plugins/tasks/networking/test_napalm_configure.py +++ b/tests/plugins/tasks/networking/test_napalm_configure.py @@ -22,7 +22,9 @@ def test_napalm_configure_change_dry_run(self, nornir): configuration = "hostname changed-hostname" d = nornir.filter(name="dev3.group_2") d.run(connect, extras=opt) - result = d.run(networking.napalm_configure, configuration=configuration) + result = d.run( + networking.napalm_configure, dry_run=True, configuration=configuration + ) assert result for h, r in result.items(): assert "+hostname changed-hostname" in r.diff @@ -62,4 +64,3 @@ def test_napalm_configure_change_error(self, nornir): processed = True assert isinstance(result.exception, exceptions.MergeConfigException) assert processed - nornir.data.reset_failed_hosts() diff --git a/tests/plugins/tasks/networking/test_napalm_get.py b/tests/plugins/tasks/networking/test_napalm_get.py index 22537ff8..0c4f9f03 100644 --- a/tests/plugins/tasks/networking/test_napalm_get.py +++ b/tests/plugins/tasks/networking/test_napalm_get.py @@ -36,7 +36,6 @@ def test_napalm_getters_error(self, nornir): processed = True assert isinstance(result.exception, KeyError) assert processed - nornir.data.reset_failed_hosts() def test_napalm_getters_with_options_error(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_single_with_options"} @@ -49,7 +48,6 @@ def test_napalm_getters_with_options_error(self, nornir): assert result.failed for h, r in result.items(): assert "unexpected keyword argument 'nonexistent'" in r.result - nornir.data.reset_failed_hosts() def test_napalm_getters_with_options_error_optional_args(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_single_with_options"} @@ -64,7 +62,6 @@ def test_napalm_getters_with_options_error_optional_args(self, nornir): assert result.failed for h, r in result.items(): assert "unexpected keyword argument 'nonexistent'" in r.result - nornir.data.reset_failed_hosts() def test_napalm_getters_single_with_options(self, nornir): opt = {"path": THIS_DIR + "/test_napalm_getters_single_with_options"} diff --git a/tests/plugins/tasks/networking/test_tcp_ping.py b/tests/plugins/tasks/networking/test_tcp_ping.py index dc9f42ea..376d4864 100644 --- a/tests/plugins/tasks/networking/test_tcp_ping.py +++ b/tests/plugins/tasks/networking/test_tcp_ping.py @@ -1,8 +1,7 @@ import os -from nornir.core import Nornir -from nornir.plugins.inventory.simple import SimpleInventory +from nornir.core import InitNornir from nornir.plugins.tasks import networking @@ -35,7 +34,6 @@ def test_tcp_ping_invalid_port(self, nornir): processed = True assert isinstance(result.exception, ValueError) assert processed - nornir.data.reset_failed_hosts() def test_tcp_ping_invalid_ports(self, nornir): results = nornir.run(networking.tcp_ping, ports=[22, "web", 443]) @@ -44,18 +42,18 @@ def test_tcp_ping_invalid_ports(self, nornir): processed = True assert isinstance(result.exception, ValueError) assert processed - nornir.data.reset_failed_hosts() + def test_tcp_ping_external_hosts(self): + external = InitNornir( + inventory={"options": {"host_file": ext_inv_file}}, dry_run=True + ) + result = external.run(networking.tcp_ping, ports=[23, 443]) -def test_tcp_ping_external_hosts(): - external = Nornir(inventory=SimpleInventory(ext_inv_file, ""), dry_run=True) - result = external.run(networking.tcp_ping, ports=[23, 443]) - - assert result - for h, r in result.items(): - if h == "www.github.com": - assert r.result[23] is False - assert r.result[443] - else: - assert r.result[23] is False - assert r.result[443] + assert result + for h, r in result.items(): + if h == "www.github.com": + assert r.result[23] is False + assert r.result[443] + else: + assert r.result[23] is False + assert r.result[443] diff --git a/tests/plugins/tasks/text/test_template_file.py b/tests/plugins/tasks/text/test_template_file.py index 3aa11501..afe0e330 100644 --- a/tests/plugins/tasks/text/test_template_file.py +++ b/tests/plugins/tasks/text/test_template_file.py @@ -30,4 +30,3 @@ def test_template_file_error_broken_file(self, nornir): processed = True assert isinstance(result.exception, TemplateSyntaxError) assert processed - nornir.data.reset_failed_hosts() diff --git a/tests/plugins/tasks/text/test_template_string.py b/tests/plugins/tasks/text/test_template_string.py index 94731f44..920970c9 100644 --- a/tests/plugins/tasks/text/test_template_string.py +++ b/tests/plugins/tasks/text/test_template_string.py @@ -44,4 +44,3 @@ def test_template_string_error_broken_string(self, nornir): processed = True assert isinstance(result.exception, TemplateSyntaxError) assert processed - nornir.data.reset_failed_hosts() From e2fec9c7acd1cf0c4a9339876d234798e31fa83f Mon Sep 17 00:00:00 2001 From: David Barroso Date: Mon, 3 Sep 2018 12:59:27 +0200 Subject: [PATCH 050/109] fix defaults --- nornir/core/inventory.py | 13 ++++++++----- tests/core/test_inventory.py | 6 ++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 1ccb6fdc..60ba61f8 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -361,26 +361,28 @@ def __init__( **kwargs, ): groups = groups or {} - defaults = defaults or {} - defaults = Defaults(**defaults) + + if defaults is None: + defaults = Defaults() parsed_hosts = {} for n, h in hosts.items(): if isinstance(h, Host): parsed_hosts[n] = h else: - parsed_hosts[n] = Host(name=n, defaults=defaults, **h) + parsed_hosts[n] = Host(name=n, **h) parsed_groups = {} for n, g in groups.items(): if isinstance(h, Host): parsed_groups[n] = g else: - parsed_groups[n] = Group(name=n, defaults=defaults, **g) + parsed_groups[n] = Group(name=n, **g) super().__init__( hosts=parsed_hosts, groups=parsed_groups, defaults=defaults, *args, **kwargs ) for n, h in parsed_hosts.items(): + h.defaults = self.defaults for p in h.groups: h.groups.refs.append(parsed_groups[p]) for n, g in parsed_groups.items(): @@ -388,6 +390,7 @@ def __init__( g.groups.refs.append(parsed_groups[p]) if transform_function: + h.defaults = self.defaults for h in self.hosts.values(): transform_function(h) @@ -401,7 +404,7 @@ def filter(self, filter_obj=None, filter_func=None, *args, **kwargs): for n, h in self.hosts.items() if all(h.get(k) == v for k, v in kwargs.items()) } - return Inventory(hosts=filtered, groups=self.groups) + return Inventory(hosts=filtered, groups=self.groups, defaults=self.defaults) def __len__(self): return self.hosts.__len__() diff --git a/tests/core/test_inventory.py b/tests/core/test_inventory.py index f81a4656..82c9453b 100644 --- a/tests/core/test_inventory.py +++ b/tests/core/test_inventory.py @@ -178,3 +178,9 @@ def test_get_connection_parameters(self): "platform": None, "extras": {"blah": "from_defaults"}, } + + def test_defaults(self): + inv = Inventory(**inv_dict) + inv.defaults.password = "asd" + assert inv.defaults.password == "asd" + assert inv.hosts["dev2.group_1"].password == "asd" From 53978f1f139ac9396b6d4b3f33582d3cae99662b Mon Sep 17 00:00:00 2001 From: David Barroso Date: Mon, 3 Sep 2018 13:00:03 +0200 Subject: [PATCH 051/109] fix howtos --- docs/howto/advanced_filtering.ipynb | 106 +++++++++-------- docs/howto/advanced_filtering/config.yaml | 9 +- .../advanced_filtering/inventory/groups.yaml | 16 +-- .../advanced_filtering/inventory/hosts.yaml | 90 +++++++------- docs/howto/handling_connections.ipynb | 26 ++-- docs/howto/handling_connections/config.yaml | 9 +- .../inventory/groups.yaml | 2 - .../handling_connections/inventory/hosts.yaml | 11 +- docs/howto/transforming_inventory_data.ipynb | 111 ++++++++++++++---- .../transforming_inventory_data/config.yaml | 11 +- .../transforming_inventory_data/helpers.py | 2 +- .../inventory/groups.yaml | 3 +- .../inventory/hosts.yaml | 6 +- .../writing_a_custom_inventory_plugin.ipynb | 81 +++++++++---- .../my_inventory.py | 18 ++- 15 files changed, 303 insertions(+), 198 deletions(-) delete mode 100644 docs/howto/handling_connections/inventory/groups.yaml diff --git a/docs/howto/advanced_filtering.ipynb b/docs/howto/advanced_filtering.ipynb index de838f99..4e4651d9 100644 --- a/docs/howto/advanced_filtering.ipynb +++ b/docs/howto/advanced_filtering.ipynb @@ -32,76 +32,82 @@ "text": [ "---\r\n", "cat:\r\n", - " domestic: true\r\n", - " diet: omnivore\r\n", " groups:\r\n", " - terrestrial\r\n", " - mammal\r\n", - " additional_data:\r\n", - " lifespan: 17\r\n", - " famous_members:\r\n", - " - garfield\r\n", - " - felix\r\n", - " - grumpy\r\n", + " data:\r\n", + " domestic: true\r\n", + " diet: omnivore\r\n", + " additional_data:\r\n", + " lifespan: 17\r\n", + " famous_members:\r\n", + " - garfield\r\n", + " - felix\r\n", + " - grumpy\r\n", "\r\n", "bat:\r\n", - " domestic: false\r\n", - " fly: true\r\n", - " diet: carnivore\r\n", " groups:\r\n", " - terrestrial\r\n", " - mammal\r\n", - " additional_data:\r\n", - " lifespan: 15\r\n", - " famous_members:\r\n", - " - batman\r\n", - " - count chocula\r\n", - " - nosferatu\r\n", + " data:\r\n", + " domestic: false\r\n", + " fly: true\r\n", + " diet: carnivore\r\n", + " additional_data:\r\n", + " lifespan: 15\r\n", + " famous_members:\r\n", + " - batman\r\n", + " - count chocula\r\n", + " - nosferatu\r\n", "\r\n", "eagle:\r\n", - " domestic: false\r\n", - " diet: carnivore\r\n", " groups:\r\n", " - terrestrial\r\n", " - bird\r\n", - " additional_data:\r\n", - " lifespan: 50\r\n", - " famous_members:\r\n", - " - thorondor\r\n", - " - sam\r\n", + " data:\r\n", + " domestic: false\r\n", + " diet: carnivore\r\n", + " additional_data:\r\n", + " lifespan: 50\r\n", + " famous_members:\r\n", + " - thorondor\r\n", + " - sam\r\n", "\r\n", "canary:\r\n", - " domestic: true\r\n", - " diet: herbivore\r\n", " groups:\r\n", " - terrestrial\r\n", " - bird\r\n", - " additional_data:\r\n", - " lifespan: 15\r\n", - " famous_members:\r\n", - " - tweetie\r\n", + " data:\r\n", + " domestic: true\r\n", + " diet: herbivore\r\n", + " additional_data:\r\n", + " lifespan: 15\r\n", + " famous_members:\r\n", + " - tweetie\r\n", "\r\n", "caterpillaer:\r\n", - " domestic: false\r\n", - " diet: herbivore\r\n", " groups:\r\n", " - terrestrial\r\n", " - invertebrate\r\n", - " additional_data:\r\n", - " lifespan: 1\r\n", - " famous_members:\r\n", - " - Hookah-Smoking\r\n", + " data:\r\n", + " domestic: false\r\n", + " diet: herbivore\r\n", + " additional_data:\r\n", + " lifespan: 1\r\n", + " famous_members:\r\n", + " - Hookah-Smoking\r\n", "\r\n", "octopus:\r\n", - " domestic: false\r\n", - " diet: carnivore\r\n", " groups:\r\n", " - marine\r\n", " - invertebrate\r\n", - " additional_data:\r\n", - " lifespan: 1\r\n", - " famous_members:\r\n", - " - sharktopus\r\n" + " data:\r\n", + " domestic: false\r\n", + " diet: carnivore\r\n", + " additional_data:\r\n", + " lifespan: 1\r\n", + " famous_members:\r\n", + " - sharktopus\r\n" ] } ], @@ -119,18 +125,20 @@ "output_type": "stream", "text": [ "---\r\n", - "defaults: {}\r\n", "mammal:\r\n", - " reproduction: birth\r\n", - " fly: false\r\n", + " data:\r\n", + " reproduction: birth\r\n", + " fly: false\r\n", "\r\n", "bird:\r\n", - " reproduction: eggs\r\n", - " fly: true\r\n", + " data:\r\n", + " reproduction: eggs\r\n", + " fly: true\r\n", "\r\n", "invertebrate:\r\n", - " reproduction: mitosis\r\n", - " fly: false\r\n", + " data:\r\n", + " reproduction: mitosis\r\n", + " fly: false\r\n", "\r\n", "terrestrial: {}\r\n", "marine: {}\r\n" diff --git a/docs/howto/advanced_filtering/config.yaml b/docs/howto/advanced_filtering/config.yaml index fdefe2a6..25cd0ad1 100644 --- a/docs/howto/advanced_filtering/config.yaml +++ b/docs/howto/advanced_filtering/config.yaml @@ -1,6 +1,7 @@ --- -inventory: nornir.plugins.inventory.simple.SimpleInventory -SimpleInventory: - host_file: "advanced_filtering/inventory/hosts.yaml" - group_file: "advanced_filtering/inventory/groups.yaml" +inventory: + plugin: nornir.plugins.inventory.simple.SimpleInventory + options: + host_file: "advanced_filtering/inventory/hosts.yaml" + group_file: "advanced_filtering/inventory/groups.yaml" diff --git a/docs/howto/advanced_filtering/inventory/groups.yaml b/docs/howto/advanced_filtering/inventory/groups.yaml index 60f4fbf8..175bae52 100644 --- a/docs/howto/advanced_filtering/inventory/groups.yaml +++ b/docs/howto/advanced_filtering/inventory/groups.yaml @@ -1,16 +1,18 @@ --- -defaults: {} mammal: - reproduction: birth - fly: false + data: + reproduction: birth + fly: false bird: - reproduction: eggs - fly: true + data: + reproduction: eggs + fly: true invertebrate: - reproduction: mitosis - fly: false + data: + reproduction: mitosis + fly: false terrestrial: {} marine: {} diff --git a/docs/howto/advanced_filtering/inventory/hosts.yaml b/docs/howto/advanced_filtering/inventory/hosts.yaml index 48fdd6de..e5d3acbb 100644 --- a/docs/howto/advanced_filtering/inventory/hosts.yaml +++ b/docs/howto/advanced_filtering/inventory/hosts.yaml @@ -1,72 +1,78 @@ --- cat: - domestic: true - diet: omnivore groups: - terrestrial - mammal - additional_data: - lifespan: 17 - famous_members: - - garfield - - felix - - grumpy + data: + domestic: true + diet: omnivore + additional_data: + lifespan: 17 + famous_members: + - garfield + - felix + - grumpy bat: - domestic: false - fly: true - diet: carnivore groups: - terrestrial - mammal - additional_data: - lifespan: 15 - famous_members: - - batman - - count chocula - - nosferatu + data: + domestic: false + fly: true + diet: carnivore + additional_data: + lifespan: 15 + famous_members: + - batman + - count chocula + - nosferatu eagle: - domestic: false - diet: carnivore groups: - terrestrial - bird - additional_data: - lifespan: 50 - famous_members: - - thorondor - - sam + data: + domestic: false + diet: carnivore + additional_data: + lifespan: 50 + famous_members: + - thorondor + - sam canary: - domestic: true - diet: herbivore groups: - terrestrial - bird - additional_data: - lifespan: 15 - famous_members: - - tweetie + data: + domestic: true + diet: herbivore + additional_data: + lifespan: 15 + famous_members: + - tweetie caterpillaer: - domestic: false - diet: herbivore groups: - terrestrial - invertebrate - additional_data: - lifespan: 1 - famous_members: - - Hookah-Smoking + data: + domestic: false + diet: herbivore + additional_data: + lifespan: 1 + famous_members: + - Hookah-Smoking octopus: - domestic: false - diet: carnivore groups: - marine - invertebrate - additional_data: - lifespan: 1 - famous_members: - - sharktopus + data: + domestic: false + diet: carnivore + additional_data: + lifespan: 1 + famous_members: + - sharktopus diff --git a/docs/howto/handling_connections.ipynb b/docs/howto/handling_connections.ipynb index d8912a2f..166bf521 100644 --- a/docs/howto/handling_connections.ipynb +++ b/docs/howto/handling_connections.ipynb @@ -82,21 +82,17 @@ "text": [ "\u001b[1m\u001b[36mtask_manages_connection_manually************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[34m* rtr00 ** changed : False *****************************************************\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[32mvvvv task_manages_connection_manually ** changed : False vvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[32m---- napalm_get ** changed : False --------------------------------------------- INFO\u001b[0m\n", - "\u001b[0m{\u001b[0m \u001b[0m'facts'\u001b[0m: \u001b[0m{\u001b[0m \u001b[0m'fqdn'\u001b[0m: \u001b[0m'localhost'\u001b[0m,\n", - " \u001b[0m'hostname'\u001b[0m: \u001b[0m'localhost'\u001b[0m,\n", - " \u001b[0m'interface_list'\u001b[0m: \u001b[0m[\u001b[0m \u001b[0m\u001b[0m'Ethernet1'\u001b[0m,\n", - " \u001b[0m'Ethernet2'\u001b[0m,\n", - " \u001b[0m'Ethernet3'\u001b[0m,\n", - " \u001b[0m'Ethernet4'\u001b[0m,\n", - " \u001b[0m'Management1'\u001b[0m]\u001b[0m,\n", - " \u001b[0m'model'\u001b[0m: \u001b[0m'vEOS'\u001b[0m,\n", - " \u001b[0m'os_version'\u001b[0m: \u001b[0m'4.15.5M-3054042.4155M'\u001b[0m,\n", - " \u001b[0m'serial_number'\u001b[0m: \u001b[0m''\u001b[0m,\n", - " \u001b[0m'uptime'\u001b[0m: \u001b[0m'...'\u001b[0m,\n", - " \u001b[0m'vendor'\u001b[0m: \u001b[0m'Arista'\u001b[0m}\u001b[0m}\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[32m^^^^ END task_manages_connection_manually ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[31mvvvv task_manages_connection_manually ** changed : False vvvvvvvvvvvvvvvvvvvvvvv ERROR\u001b[0m\n", + "\u001b[0mTraceback (most recent call last):\n", + " File \"/Users/dbarroso/workspace/nornir/nornir/core/task.py\", line 63, in start\n", + " r = self.task(self, **self.params)\n", + " File \"\", line 5, in task_manages_connection_manually\n", + " getters=[\"facts\"]\n", + " File \"/Users/dbarroso/workspace/nornir/nornir/core/task.py\", line 100, in run\n", + " raise Exception(msg)\n", + "Exception: ('You have to call this after setting host and nornir attributes. ', 'You probably called this from outside a nested task')\n", + "\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[31m^^^^ END task_manages_connection_manually ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", "\u001b[0m" ] } diff --git a/docs/howto/handling_connections/config.yaml b/docs/howto/handling_connections/config.yaml index 19d67fcb..e50d6422 100644 --- a/docs/howto/handling_connections/config.yaml +++ b/docs/howto/handling_connections/config.yaml @@ -1,6 +1,7 @@ --- -inventory: nornir.plugins.inventory.simple.SimpleInventory -SimpleInventory: - host_file: "handling_connections/inventory/hosts.yaml" - group_file: "handling_connections/inventory/groups.yaml" +inventory: + plugin: nornir.plugins.inventory.simple.SimpleInventory + options: + host_file: "handling_connections/inventory/hosts.yaml" + group_file: "handling_connections/inventory/groups.yaml" diff --git a/docs/howto/handling_connections/inventory/groups.yaml b/docs/howto/handling_connections/inventory/groups.yaml deleted file mode 100644 index dfdd45e6..00000000 --- a/docs/howto/handling_connections/inventory/groups.yaml +++ /dev/null @@ -1,2 +0,0 @@ ---- -defaults: {} diff --git a/docs/howto/handling_connections/inventory/hosts.yaml b/docs/howto/handling_connections/inventory/hosts.yaml index 5820cdd5..433a179e 100644 --- a/docs/howto/handling_connections/inventory/hosts.yaml +++ b/docs/howto/handling_connections/inventory/hosts.yaml @@ -1,7 +1,8 @@ --- rtr00: - platform: mock - napalm_options: - connection_options: - optional_args: - path: handling_connections/mocked_data + connection_options: + napalm: + platform: mock + extras: + optional_args: + path: handling_connections/mocked_data diff --git a/docs/howto/transforming_inventory_data.ipynb b/docs/howto/transforming_inventory_data.ipynb index 653b959d..0781ea35 100644 --- a/docs/howto/transforming_inventory_data.ipynb +++ b/docs/howto/transforming_inventory_data.ipynb @@ -20,9 +20,11 @@ "text": [ "---\r\n", "rtr00:\r\n", - " user: automation_user\r\n", + " data:\r\n", + " user: automation_user\r\n", "rtr01:\r\n", - " user: automation_user\r\n" + " data:\r\n", + " user: automation_user\r\n" ] } ], @@ -65,12 +67,36 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'groups': {'defaults': {'a_default_attribute': 'my_default'}},\n", - " 'hosts': {'rtr00': {'name': 'rtr00',\n", - " 'user': 'automation_user',\n", + "{'defaults': {'connection_options': {},\n", + " 'data': {},\n", + " 'hostname': None,\n", + " 'password': None,\n", + " 'platform': None,\n", + " 'port': None,\n", + " 'username': None},\n", + " 'groups': {'defaults': {'connection_options': {},\n", + " 'data': {'a_default_attribute': 'my_default'},\n", + " 'groups': [],\n", + " 'hostname': None,\n", + " 'password': None,\n", + " 'platform': None,\n", + " 'port': None,\n", + " 'username': None}},\n", + " 'hosts': {'rtr00': {'connection_options': {},\n", + " 'data': {'user': 'automation_user'},\n", + " 'groups': [],\n", + " 'hostname': None,\n", + " 'password': None,\n", + " 'platform': None,\n", + " 'port': None,\n", " 'username': 'automation_user'},\n", - " 'rtr01': {'name': 'rtr01',\n", - " 'user': 'automation_user',\n", + " 'rtr01': {'connection_options': {},\n", + " 'data': {'user': 'automation_user'},\n", + " 'groups': [],\n", + " 'hostname': None,\n", + " 'password': None,\n", + " 'platform': None,\n", + " 'port': None,\n", " 'username': 'automation_user'}}}\n" ] } @@ -81,13 +107,19 @@ "\n", "def adapt_host_data(host):\n", " # This function receives a Host object for manipulation\n", - " host[\"username\"] = host[\"user\"]\n", + " host.username = host[\"user\"]\n", "\n", "nr = InitNornir(\n", - " config_file=\"transforming_inventory_data/config.yaml\",\n", - " transform_function=adapt_host_data,\n", + " inventory={\n", + " \"plugin\": \"nornir.plugins.inventory.simple.SimpleInventory\",\n", + " \"options\": {\n", + " \"host_file\": \"transforming_inventory_data/inventory/hosts.yaml\",\n", + " \"group_file\": \"transforming_inventory_data/inventory/groups.yaml\",\n", + " },\n", + " \"transform_function\": adapt_host_data,\n", + " },\n", ")\n", - "pprint.pprint(nr.inventory.to_dict())" + "pprint.pprint(nr.inventory.dict())" ] }, { @@ -110,7 +142,7 @@ "text": [ "def adapt_host_data(host):\r\n", " # This function receives a Host object for manipulation\r\n", - " host[\"username\"] = host[\"user\"]\r\n" + " host.username = host[\"user\"]\r\n" ] } ], @@ -163,11 +195,12 @@ "output_type": "stream", "text": [ "---\r\n", - "inventory: nornir.plugins.inventory.simple.SimpleInventory\r\n", - "SimpleInventory:\r\n", - " host_file: \"transforming_inventory_data/inventory/hosts.yaml\"\r\n", - " group_file: \"transforming_inventory_data/inventory/groups.yaml\"\r\n", - "transform_function: \"transforming_inventory_data.helpers.adapt_host_data\"\r\n" + "inventory:\r\n", + " plugin: nornir.plugins.inventory.simple.SimpleInventory\r\n", + " options:\r\n", + " host_file: \"transforming_inventory_data/inventory/hosts.yaml\"\r\n", + " group_file: \"transforming_inventory_data/inventory/groups.yaml\"\r\n", + " transform_function: \"transforming_inventory_data.helpers.adapt_host_data\"\r\n" ] } ], @@ -191,12 +224,36 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'groups': {'defaults': {'a_default_attribute': 'my_default'}},\n", - " 'hosts': {'rtr00': {'name': 'rtr00',\n", - " 'user': 'automation_user',\n", + "{'defaults': {'connection_options': {},\n", + " 'data': {},\n", + " 'hostname': None,\n", + " 'password': None,\n", + " 'platform': None,\n", + " 'port': None,\n", + " 'username': None},\n", + " 'groups': {'defaults': {'connection_options': {},\n", + " 'data': {'a_default_attribute': 'my_default'},\n", + " 'groups': [],\n", + " 'hostname': None,\n", + " 'password': None,\n", + " 'platform': None,\n", + " 'port': None,\n", + " 'username': None}},\n", + " 'hosts': {'rtr00': {'connection_options': {},\n", + " 'data': {'user': 'automation_user'},\n", + " 'groups': [],\n", + " 'hostname': None,\n", + " 'password': None,\n", + " 'platform': None,\n", + " 'port': None,\n", " 'username': 'automation_user'},\n", - " 'rtr01': {'name': 'rtr01',\n", - " 'user': 'automation_user',\n", + " 'rtr01': {'connection_options': {},\n", + " 'data': {'user': 'automation_user'},\n", + " 'groups': [],\n", + " 'hostname': None,\n", + " 'password': None,\n", + " 'platform': None,\n", + " 'port': None,\n", " 'username': 'automation_user'}}}\n" ] } @@ -208,7 +265,7 @@ "nr = InitNornir(\n", " config_file=\"transforming_inventory_data/config.yaml\",\n", ")\n", - "pprint.pprint(nr.inventory.to_dict())" + "pprint.pprint(nr.inventory.dict())" ] }, { @@ -229,7 +286,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Before setting password: \n", + "Before setting password: None\n", + "After setting password: a_secret_password\n", "After setting password: a_secret_password\n" ] } @@ -245,8 +303,9 @@ " config_file=\"transforming_inventory_data/config.yaml\",\n", ")\n", "print(\"Before setting password: \", nr.inventory.hosts[\"rtr00\"].password)\n", - "nr.inventory.defaults[\"password\"] = password\n", - "print(\"After setting password: \", nr.inventory.hosts[\"rtr00\"][\"password\"])" + "nr.inventory.defaults.password = password\n", + "print(\"After setting password: \", nr.inventory.hosts[\"rtr00\"].password)\n", + "print(\"After setting password: \", nr.inventory.defaults.password)" ] }, { diff --git a/docs/howto/transforming_inventory_data/config.yaml b/docs/howto/transforming_inventory_data/config.yaml index 8c8365e8..8471aa58 100644 --- a/docs/howto/transforming_inventory_data/config.yaml +++ b/docs/howto/transforming_inventory_data/config.yaml @@ -1,6 +1,7 @@ --- -inventory: nornir.plugins.inventory.simple.SimpleInventory -SimpleInventory: - host_file: "transforming_inventory_data/inventory/hosts.yaml" - group_file: "transforming_inventory_data/inventory/groups.yaml" -transform_function: "transforming_inventory_data.helpers.adapt_host_data" +inventory: + plugin: nornir.plugins.inventory.simple.SimpleInventory + options: + host_file: "transforming_inventory_data/inventory/hosts.yaml" + group_file: "transforming_inventory_data/inventory/groups.yaml" + transform_function: "transforming_inventory_data.helpers.adapt_host_data" diff --git a/docs/howto/transforming_inventory_data/helpers.py b/docs/howto/transforming_inventory_data/helpers.py index 4642cfa9..65471ae7 100644 --- a/docs/howto/transforming_inventory_data/helpers.py +++ b/docs/howto/transforming_inventory_data/helpers.py @@ -1,3 +1,3 @@ def adapt_host_data(host): # This function receives a Host object for manipulation - host["username"] = host["user"] + host.username = host["user"] diff --git a/docs/howto/transforming_inventory_data/inventory/groups.yaml b/docs/howto/transforming_inventory_data/inventory/groups.yaml index fe974dda..21583061 100644 --- a/docs/howto/transforming_inventory_data/inventory/groups.yaml +++ b/docs/howto/transforming_inventory_data/inventory/groups.yaml @@ -1,3 +1,4 @@ --- defaults: - a_default_attribute: my_default + data: + a_default_attribute: my_default diff --git a/docs/howto/transforming_inventory_data/inventory/hosts.yaml b/docs/howto/transforming_inventory_data/inventory/hosts.yaml index 2e49ed07..cc252a56 100644 --- a/docs/howto/transforming_inventory_data/inventory/hosts.yaml +++ b/docs/howto/transforming_inventory_data/inventory/hosts.yaml @@ -1,5 +1,7 @@ --- rtr00: - user: automation_user + data: + user: automation_user rtr01: - user: automation_user + data: + user: automation_user diff --git a/docs/howto/writing_a_custom_inventory_plugin.ipynb b/docs/howto/writing_a_custom_inventory_plugin.ipynb index e3316d69..e3d4f206 100644 --- a/docs/howto/writing_a_custom_inventory_plugin.ipynb +++ b/docs/howto/writing_a_custom_inventory_plugin.ipynb @@ -26,26 +26,24 @@ " # code to get the data\r\n", " hosts = {\r\n", " \"host1\": {\r\n", - " \"data1\": \"value1\",\r\n", - " \"data2\": \"value2\",\r\n", - " \"data3\": \"value3\",\r\n", + " \"data\": {\"data1\": \"value1\", \"data2\": \"value2\", \"data3\": \"value3\"},\r\n", " \"groups\": [\"my_group1\"],\r\n", " },\r\n", " \"host2\": {\r\n", - " \"data1\": \"value1\",\r\n", - " \"data2\": \"value2\",\r\n", - " \"data3\": \"value3\",\r\n", + " \"data\": {\"data1\": \"value1\", \"data2\": \"value2\", \"data3\": \"value3\"},\r\n", " \"groups\": [\"my_group1\"],\r\n", " },\r\n", " }\r\n", " groups = {\r\n", " \"my_group1\": {\r\n", - " \"more_data1\": \"more_value1\",\r\n", - " \"more_data2\": \"more_value2\",\r\n", - " \"more_data3\": \"more_value3\",\r\n", + " \"data\": {\r\n", + " \"more_data1\": \"more_value1\",\r\n", + " \"more_data2\": \"more_value2\",\r\n", + " \"more_data3\": \"more_value3\",\r\n", + " }\r\n", " }\r\n", " }\r\n", - " defaults = {\"location\": \"internet\", \"language\": \"Python\"}\r\n", + " defaults = {\"data\": {\"location\": \"internet\", \"language\": \"Python\"}}\r\n", "\r\n", " # passing the data to the parent class so the data is\r\n", " # transformed into actual Host/Group objects\r\n", @@ -74,21 +72,43 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'groups': {'defaults': {'language': 'Python', 'location': 'internet'},\n", - " 'my_group1': {'more_data1': 'more_value1',\n", - " 'more_data2': 'more_value2',\n", - " 'more_data3': 'more_value3',\n", - " 'name': 'my_group1'}},\n", - " 'hosts': {'host1': {'data1': 'value1',\n", - " 'data2': 'value2',\n", - " 'data3': 'value3',\n", + "{'defaults': {'connection_options': {},\n", + " 'data': {'language': 'Python', 'location': 'internet'},\n", + " 'hostname': None,\n", + " 'password': None,\n", + " 'platform': None,\n", + " 'port': None,\n", + " 'username': None},\n", + " 'groups': {'my_group1': {'connection_options': {},\n", + " 'data': {'more_data1': 'more_value1',\n", + " 'more_data2': 'more_value2',\n", + " 'more_data3': 'more_value3'},\n", + " 'groups': [],\n", + " 'hostname': None,\n", + " 'password': None,\n", + " 'platform': None,\n", + " 'port': None,\n", + " 'username': None}},\n", + " 'hosts': {'host1': {'connection_options': {},\n", + " 'data': {'data1': 'value1',\n", + " 'data2': 'value2',\n", + " 'data3': 'value3'},\n", " 'groups': ['my_group1'],\n", - " 'name': 'host1'},\n", - " 'host2': {'data1': 'value1',\n", - " 'data2': 'value2',\n", - " 'data3': 'value3',\n", + " 'hostname': None,\n", + " 'password': None,\n", + " 'platform': None,\n", + " 'port': None,\n", + " 'username': None},\n", + " 'host2': {'connection_options': {},\n", + " 'data': {'data1': 'value1',\n", + " 'data2': 'value2',\n", + " 'data3': 'value3'},\n", " 'groups': ['my_group1'],\n", - " 'name': 'host2'}}}\n" + " 'hostname': None,\n", + " 'password': None,\n", + " 'platform': None,\n", + " 'port': None,\n", + " 'username': None}}}\n" ] } ], @@ -96,9 +116,20 @@ "from nornir.core import InitNornir\n", "import pprint\n", "\n", - "nr = InitNornir(inventory=\"writing_a_custom_inventory_plugin.my_inventory.MyInventory\")\n", - "pprint.pprint(nr.inventory.to_dict())" + "nr = InitNornir(\n", + " inventory={\n", + " \"plugin\": \"writing_a_custom_inventory_plugin.my_inventory.MyInventory\"\n", + " }\n", + ")\n", + "pprint.pprint(nr.inventory.dict())" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/docs/howto/writing_a_custom_inventory_plugin/my_inventory.py b/docs/howto/writing_a_custom_inventory_plugin/my_inventory.py index b0787e23..5f747d04 100644 --- a/docs/howto/writing_a_custom_inventory_plugin/my_inventory.py +++ b/docs/howto/writing_a_custom_inventory_plugin/my_inventory.py @@ -6,26 +6,24 @@ def __init__(self, **kwargs): # code to get the data hosts = { "host1": { - "data1": "value1", - "data2": "value2", - "data3": "value3", + "data": {"data1": "value1", "data2": "value2", "data3": "value3"}, "groups": ["my_group1"], }, "host2": { - "data1": "value1", - "data2": "value2", - "data3": "value3", + "data": {"data1": "value1", "data2": "value2", "data3": "value3"}, "groups": ["my_group1"], }, } groups = { "my_group1": { - "more_data1": "more_value1", - "more_data2": "more_value2", - "more_data3": "more_value3", + "data": { + "more_data1": "more_value1", + "more_data2": "more_value2", + "more_data3": "more_value3", + } } } - defaults = {"location": "internet", "language": "Python"} + defaults = {"data": {"location": "internet", "language": "Python"}} # passing the data to the parent class so the data is # transformed into actual Host/Group objects From 2720a0f32cd9ba9275c0ddcc8caf618219780770 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Mon, 3 Sep 2018 13:10:59 +0200 Subject: [PATCH 052/109] undo GlobalState as a class object --- nornir/core/__init__.py | 13 +++++++------ nornir/core/inventory.py | 4 ---- nornir/core/state.py | 20 +++++++++----------- nornir/core/task.py | 3 +-- setup.cfg | 2 +- tests/conftest.py | 6 +++++- 6 files changed, 23 insertions(+), 25 deletions(-) diff --git a/nornir/core/__init__.py b/nornir/core/__init__.py index 4ee13ed3..e0af8687 100644 --- a/nornir/core/__init__.py +++ b/nornir/core/__init__.py @@ -17,7 +17,7 @@ class Nornir(object): Arguments: inventory (:obj:`nornir.core.inventory.Inventory`): Inventory to work with - data(:obj:`nornir.core.GlobalState`): shared data amongst different iterations of nornir + data(GlobalState): shared data amongst different iterations of nornir dry_run(``bool``): Whether if we are testing the changes or not config (:obj:`nornir.core.configuration.Config`): Configuration object config_file (``str``): Path to Yaml configuration file @@ -32,6 +32,7 @@ class Nornir(object): def __init__( self, inventory, _config=None, config_file=None, logger=None, data=None ): + self.data = data if data is not None else GlobalState() self.logger = logger or logging.getLogger("nornir") self.inventory = inventory @@ -121,11 +122,11 @@ def run( run_on = [] if on_good: for name, host in self.inventory.hosts.items(): - if name not in GlobalState.failed_hosts: + if name not in self.data.failed_hosts: run_on.append(host) if on_failed: for name, host in self.inventory.hosts.items(): - if name in GlobalState.failed_hosts: + if name in self.data.failed_hosts: run_on.append(host) self.logger.info( @@ -146,12 +147,12 @@ def run( if raise_on_error: result.raise_on_error() else: - GlobalState.failed_hosts.update(result.failed_hosts.keys()) + self.data.failed_hosts.update(result.failed_hosts.keys()) return result - def to_dict(self): + def dict(self): """ Return a dictionary representing the object. """ - return {"data": GlobalState.to_dict(), "inventory": self.inventory.to_dict()} + return {"data": self.data.dict(), "inventory": self.inventory.dict()} def close_connections(self, on_good=True, on_failed=False): def close_connections_task(task): diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 60ba61f8..770a9290 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -110,10 +110,6 @@ def items(self): """ return self._resolve_data().items() - def to_dict(self): - """ Return a dictionary representing the object. """ - return self.data - def has_parent_group(self, group): """Retuns whether the object is a child of the :obj:`Group` ``group``""" if isinstance(group, str): diff --git a/nornir/core/state.py b/nornir/core/state.py index 5481a6e8..b10a6f4c 100644 --- a/nornir/core/state.py +++ b/nornir/core/state.py @@ -7,20 +7,18 @@ class GlobalState(object): failed_hosts (list): Hosts that have failed to run a task properly """ - dry_run = None - failed_hosts = set() + def __init__(self): + self.dry_run = None + self.failed_hosts = set() - @classmethod - def recover_host(cls, host): + def recover_host(self, host): """Remove ``host`` from list of failed hosts.""" - cls.failed_hosts.discard(host) + self.failed_hosts.discard(host) - @classmethod - def reset_failed_hosts(cls): + def reset_failed_hosts(self): """Reset failed hosts and make all hosts available for future tasks.""" - cls.failed_hosts = set() + self.failed_hosts = set() - @classmethod - def to_dict(cls): + def to_dict(self): """ Return a dictionary representing the object. """ - return cls.__dict__ + return self.__dict__ diff --git a/nornir/core/task.py b/nornir/core/task.py index 674b261a..fd266026 100644 --- a/nornir/core/task.py +++ b/nornir/core/task.py @@ -2,7 +2,6 @@ import traceback from typing import Optional -from nornir.core.state import GlobalState from nornir.core.exceptions import NornirExecutionError from nornir.core.exceptions import NornirSubTaskError @@ -118,7 +117,7 @@ def is_dry_run(self, override=None): Arguments: override (bool): Override for current task """ - return override if override is not None else GlobalState.dry_run + return override if override is not None else self.nornir.dry_run class Result(object): diff --git a/setup.cfg b/setup.cfg index ea33b94a..9be28f7d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,7 +18,7 @@ skip = .tox/* max_line_length = 100 [tool:pytest] -addopts = --cov=nornir --cov-report=term-missing -vs +#addopts = --cov=nornir --cov-report=term-missing -vs python_paths = ./ [mypy] diff --git a/tests/conftest.py b/tests/conftest.py index 48c00f61..23fed3fd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,9 @@ import pytest +global_data = GlobalState() + + logging.basicConfig( filename="tests.log", filemode="w", @@ -55,9 +58,10 @@ def nornir(request): }, dry_run=True, ) + nornir.data = global_data return nornir @pytest.fixture(scope="function", autouse=True) def reset_failed_hosts(): - return GlobalState.reset_failed_hosts() + global_data.reset_failed_hosts() From e31073521b2513a1ec1682a226c070944021e25b Mon Sep 17 00:00:00 2001 From: David Barroso Date: Tue, 4 Sep 2018 10:04:05 +0200 Subject: [PATCH 053/109] progress --- docs/howto/inventory.ipynb | 1018 ++++++++++++++++++++++++ docs/howto/inventory/defaults.yaml | 4 + docs/howto/inventory/groups.yaml | 21 + docs/howto/inventory/hosts.yaml | 162 ++++ nornir/core/__init__.py | 4 +- nornir/core/deserializer/__init__.py | 0 nornir/core/deserializer/inventory.py | 90 +++ nornir/core/inventory.py | 176 ++-- nornir/plugins/inventory/netbox.py | 2 +- tests/conftest.py | 102 +-- tests/core/test_InitNornir.py | 4 +- tests/core/test_filter.py | 2 +- tests/core/test_inventory.py | 58 +- tests/inventory_data/hosts.yaml | 1 + tests/plugins/inventory/test_netbox.py | 2 +- 15 files changed, 1476 insertions(+), 170 deletions(-) create mode 100644 docs/howto/inventory.ipynb create mode 100644 docs/howto/inventory/defaults.yaml create mode 100644 docs/howto/inventory/groups.yaml create mode 100644 docs/howto/inventory/hosts.yaml create mode 100644 nornir/core/deserializer/__init__.py create mode 100644 nornir/core/deserializer/inventory.py diff --git a/docs/howto/inventory.ipynb b/docs/howto/inventory.ipynb new file mode 100644 index 00000000..241dc9b7 --- /dev/null +++ b/docs/howto/inventory.ipynb @@ -0,0 +1,1018 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "nbsphinx": "hidden" + }, + "outputs": [], + "source": [ + "# ignore this cell, this is just a helper cell to provide the magic %highlight_file\n", + "%run ../highlighter.py" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Inventory\n", + "\n", + "The Inventory is arguably the most important piece of nornir. Let's see how it works. To begin with the [inventory](../../ref/api/inventory.rst#nornir.core.inventory.Inventory) is comprised of [hosts](../../ref/api/inventory.rst#nornir.core.inventory.Host) and [groups](../../ref/api/inventory.rst#nornir.core.inventory.Group).\n", + "\n", + "In this tutorial we are using the [SimpleInventory](../../plugins/inventory/simple.rst#nornir.plugins.inventory.simple.SimpleInventory) plugin. This inventory plugin stores all the relevant data in two files. Let's start by checking them:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
  1 ---\n",
+       "  2 host1.cmh:\n",
+       "  3     hostname: 127.0.0.1\n",
+       "  4     port: 2201\n",
+       "  5     username: vagrant\n",
+       "  6     password: vagrant\n",
+       "  7     groups:\n",
+       "  8         - cmh\n",
+       "  9     platform: linux\n",
+       " 10     data:\n",
+       " 11         site: cmh\n",
+       " 12         role: host\n",
+       " 13         type: host\n",
+       " 14         nested_data:\n",
+       " 15             a_dict:\n",
+       " 16                 a: 1\n",
+       " 17                 b: 2\n",
+       " 18             a_list: [1, 2]\n",
+       " 19             a_string: "asdasd"\n",
+       " 20 \n",
+       " 21 host2.cmh:\n",
+       " 22     hostname: 127.0.0.1\n",
+       " 23     port: 2202\n",
+       " 24     username: vagrant\n",
+       " 25     password: vagrant\n",
+       " 26     groups:\n",
+       " 27         - cmh\n",
+       " 28     platform: linux\n",
+       " 29     data:\n",
+       " 30         site: cmh\n",
+       " 31         role: host\n",
+       " 32         type: host\n",
+       " 33         nested_data:\n",
+       " 34             a_dict:\n",
+       " 35                 b: 2\n",
+       " 36                 c: 3\n",
+       " 37             a_list: [1, 2]\n",
+       " 38             a_string: "qwe"\n",
+       " 39 \n",
+       " 40 spine00.cmh:\n",
+       " 41     hostname: 127.0.0.1\n",
+       " 42     username: vagrant\n",
+       " 43     password: vagrant\n",
+       " 44     port: 12444\n",
+       " 45     platform: eos\n",
+       " 46     groups:\n",
+       " 47         - cmh\n",
+       " 48     data:\n",
+       " 49         site: cmh\n",
+       " 50         role: spine\n",
+       " 51         type: network_device\n",
+       " 52 \n",
+       " 53 spine01.cmh:\n",
+       " 54     hostname: 127.0.0.1\n",
+       " 55     username: vagrant\n",
+       " 56     password: ""\n",
+       " 57     port: 12204\n",
+       " 58     platform: junos\n",
+       " 59     groups:\n",
+       " 60         - cmh\n",
+       " 61     data:\n",
+       " 62         site: cmh\n",
+       " 63         role: spine\n",
+       " 64         type: network_device\n",
+       " 65 \n",
+       " 66 leaf00.cmh:\n",
+       " 67     hostname: 127.0.0.1\n",
+       " 68     username: vagrant\n",
+       " 69     password: vagrant\n",
+       " 70     port: 12443\n",
+       " 71     groups:\n",
+       " 72         - cmh\n",
+       " 73     platform: eos\n",
+       " 74     data:\n",
+       " 75         site: cmh\n",
+       " 76         role: leaf\n",
+       " 77         type: network_device\n",
+       " 78         asn: 65100\n",
+       " 79 \n",
+       " 80 leaf01.cmh:\n",
+       " 81     hostname: 127.0.0.1\n",
+       " 82     username: vagrant\n",
+       " 83     password: ""\n",
+       " 84     port: 12203\n",
+       " 85     groups:\n",
+       " 86         - cmh\n",
+       " 87     platform: junos\n",
+       " 88     data:\n",
+       " 89         site: cmh\n",
+       " 90         role: leaf\n",
+       " 91         type: network_device\n",
+       " 92         asn: 65101\n",
+       " 93 \n",
+       " 94 host1.bma:\n",
+       " 95     groups:\n",
+       " 96         - bma\n",
+       " 97     platform: linux\n",
+       " 98     data:\n",
+       " 99         type: host\n",
+       "100         site: bma\n",
+       "101         role: host\n",
+       "102 \n",
+       "103 host2.bma:\n",
+       "104     groups:\n",
+       "105         - bma\n",
+       "106     platform: linux\n",
+       "107     data:\n",
+       "108         type: host\n",
+       "109         site: bma\n",
+       "110         role: host\n",
+       "111 \n",
+       "112 spine00.bma:\n",
+       "113     hostname: 127.0.0.1\n",
+       "114     username: vagrant\n",
+       "115     password: vagrant\n",
+       "116     port: 12444\n",
+       "117     groups:\n",
+       "118         - bma\n",
+       "119     platform: eos\n",
+       "120     data:\n",
+       "121         site: bma\n",
+       "122         role: spine\n",
+       "123         type: network_device\n",
+       "124 \n",
+       "125 spine01.bma:\n",
+       "126     hostname: 127.0.0.1\n",
+       "127     username: vagrant\n",
+       "128     password: ""\n",
+       "129     port: 12204\n",
+       "130     groups:\n",
+       "131         - bma\n",
+       "132     platform: junos\n",
+       "133     data:\n",
+       "134         type: network_device\n",
+       "135         site: bma\n",
+       "136         role: spine\n",
+       "137 \n",
+       "138 leaf00.bma:\n",
+       "139     hostname: 127.0.0.1\n",
+       "140     username: vagrant\n",
+       "141     password: vagrant\n",
+       "142     port: 12443\n",
+       "143     groups:\n",
+       "144         - bma\n",
+       "145     platform: eos\n",
+       "146     data:\n",
+       "147         type: network_device\n",
+       "148         site: bma\n",
+       "149         role: leaf\n",
+       "150 \n",
+       "151 leaf01.bma:\n",
+       "152     hostname: 127.0.0.1\n",
+       "153     username: vagrant\n",
+       "154     password: wrong_password\n",
+       "155     port: 12203\n",
+       "156     groups:\n",
+       "157         - bma\n",
+       "158     platform: junos\n",
+       "159     data:\n",
+       "160         type: network_device\n",
+       "161         site: bma\n",
+       "162         role: leaf\n",
+       "
\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# hosts file\n", + "%highlight_file inventory/hosts.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The hosts file is basically a map where the outermost key is the hostname and then any arbitrary `` pair you want inside. Some keys like hostname, username or password have special meaning, you can investigate the [hosts](../../ref/api/inventory.rst#nornir.core.inventory.Host) class for details on those. In addition, the `groups` key is a list of groups you can inherite data from. We will inspect soon how the inheritance model works.\n", + "\n", + "Now, let's look at the groups file:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
 1 ---\n",
+       " 2 global:\n",
+       " 3     data:\n",
+       " 4         domain: global.local\n",
+       " 5         asn: 1\n",
+       " 6 \n",
+       " 7 eu:\n",
+       " 8     data:\n",
+       " 9         asn: 65100\n",
+       "10 \n",
+       "11 bma:\n",
+       "12     groups:\n",
+       "13         - eu\n",
+       "14         - global\n",
+       "15 \n",
+       "16 cmh:\n",
+       "17     data:\n",
+       "18         asn: 65000\n",
+       "19         vlans:\n",
+       "20           100: frontend\n",
+       "21           200: backend\n",
+       "
\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# groups file\n", + "%highlight_file inventory/groups.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
1 ---\n",
+       "2 username: admin\n",
+       "3 data:\n",
+       "4     domain: acme.local\n",
+       "
\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# defaults\n", + "%highlight_file inventory/defaults.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'properties': {'defaults': {'default': ,\n", + " 'properties': {...},\n", + " 'required': False,\n", + " 'title': 'Defaults',\n", + " 'type': 'object'},\n", + " 'groups': {'default': {},\n", + " 'required': False,\n", + " 'title': 'Groups',\n", + " 'type': 'Groups'},\n", + " 'hosts': {'required': True, 'title': 'Hosts', 'type': 'Hosts'}},\n", + " 'title': 'Inventory',\n", + " 'type': 'object'}\n" + ] + } + ], + "source": [ + "from nornir.core.inventory import Inventory\n", + "import pprint\n", + "pprint.pprint(Inventory.schema(), depth=3)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "ename": "Exception", + "evalue": "666", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mException\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m666\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mException\u001b[0m: 666" + ] + } + ], + "source": [ + "raise Exception(666)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Pretty similar to the hosts file.\n", + "\n", + "### Accessing the inventory\n", + "\n", + "You can access the [inventory](../../ref/api/inventory.rst#nornir.core.inventory.Inventory) with the `inventory` attribute:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from nornir.core import InitNornir\n", + "nr = InitNornir(\n", + " inventory={\n", + " \"options\": {\n", + " \"host_file\": \"inventory/hosts.yaml\",\n", + " \"group_file\": \"inventory/groups.yaml\",\n", + " \"defaults_file\": \"inventory/defaults.yaml\",\n", + " }\n", + " }\n", + ")\n", + "\n", + "nr.inventory" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The inventory has two dict-like attributes `hosts` and `groups` that you can use to access the hosts and groups respectively:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nr.inventory.hosts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nr.inventory.groups" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nr.inventory.hosts[\"leaf01.bma\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Hosts and groups are also dict-like objects:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "host = nr.inventory.hosts[\"leaf01.bma\"]\n", + "host.keys()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "host[\"site\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Inheritance model\n", + "\n", + "Let's see how the inheritance models works by example. Let's start by looking again at the groups file:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# groups file\n", + "%highlight_file inventory/groups.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The host `leaf01.bma` belongs to the group `bma` which in turn belongs to the groups `eu` and `global`. The host `spine00.cmh` belongs to the group `cmh` which doesn't belong to any other group.\n", + "\n", + "Data resolution works by iterating recursively over all the parent groups and trying to see if that parent group (or any of it's parents) contains the data. For instance:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "leaf01_bma = nr.inventory.hosts[\"leaf01.bma\"]\n", + "leaf01_bma[\"domain\"] # comes from the group `global`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "leaf01_bma[\"asn\"] # comes from group `eu`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The group `defaults` is special. This group contains data that will be returned if neither the host nor the parents have a specific value for it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "leaf01_cmh = nr.inventory.hosts[\"leaf01.cmh\"]\n", + "leaf01_cmh[\"domain\"] # comes from defaults" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If nornir can't resolve the data you should get a KeyError as usual:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " leaf01_cmh[\"non_existent\"]\n", + "except KeyError as e:\n", + " print(f\"Couldn't find key: {e}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also try to access data without recursive resolution by using the `data` attribute. For example, if we try to access `leaf01_cmh.data[\"domain\"]` we should get an error as the host itself doesn't have that data:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " leaf01_cmh.data[\"domain\"]\n", + "except KeyError as e:\n", + " print(f\"Couldn't find key: {e}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Filtering the inventory\n", + "\n", + "So far we have seen that `nr.inventory.hosts` and `nr.inventory.groups` are dict-like objects that we can use to iterate over all the hosts and groups or to access any particular one directly. Now we are going to see how we can do some fancy filtering that will enable us to operate on groups of hosts based on their properties.\n", + "\n", + "The simpler way of filtering hosts is by `` pairs. For instance:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nr.filter(site=\"cmh\").inventory.hosts.keys()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also filter using multiple `` pairs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nr.filter(site=\"cmh\", role=\"spine\").inventory.hosts.keys()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Filter is cumulative:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nr.filter(site=\"cmh\").filter(role=\"spine\").inventory.hosts.keys()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Or:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cmh = nr.filter(site=\"cmh\")\n", + "cmh.filter(role=\"spine\").inventory.hosts.keys()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cmh.filter(role=\"leaf\").inventory.hosts.keys()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also grab the children of a group:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Advanced filtering\n", + "\n", + "Sometimes you need more fancy filtering. For those cases you have two options:\n", + "\n", + "1. Use a filter function.\n", + "2. Use a filter object.\n", + "\n", + "##### Filter functions\n", + "\n", + "The ``filter_func`` parameter let's you run your own code to filter the hosts. The function signature is as simple as ``my_func(host)`` where host is an object of type [Host](../../ref/api/inventory.rst#nornir.core.inventory.Host) and it has to return either ``True`` or ``False`` to indicate if you want to host or not." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def has_long_name(host):\n", + " return len(host.name) == 11\n", + "\n", + "nr.filter(filter_func=has_long_name).inventory.hosts.keys()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Or a lambda function\n", + "nr.filter(filter_func=lambda h: len(h.name) == 9).inventory.hosts.keys()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Filter Object\n", + "\n", + "You can also use a filter object to create incrementally a complext query object. Let's see how it works by example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# first you need to import the F object\n", + "from nornir.core.filter import F" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# hosts in group cmh\n", + "cmh = nr.filter(F(groups__contains=\"cmh\"))\n", + "print(cmh.inventory.hosts.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# devices running either linux or eos\n", + "linux_or_eos = nr.filter(F(platform=\"linux\") | F(platform=\"eos\"))\n", + "print(linux_or_eos.inventory.hosts.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# spines in cmh\n", + "cmh_and_spine = nr.filter(F(groups__contains=\"cmh\") & F(role=\"spine\"))\n", + "print(cmh_and_spine.inventory.hosts.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# cmh devices that are not spines\n", + "cmh_and_not_spine = nr.filter(F(groups__contains=\"cmh\") & ~F(role=\"spine\"))\n", + "print(cmh_and_not_spine.inventory.hosts.keys())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also access nested data and even check if dicts/lists/strings contains elements. Again, let's see by example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nested_string_asd = nr.filter(F(nested_data__a_string__contains=\"asd\"))\n", + "print(nested_string_asd.inventory.hosts.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "a_dict_element_equals = nr.filter(F(nested_data__a_dict__c=3))\n", + "print(a_dict_element_equals.inventory.hosts.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "a_list_contains = nr.filter(F(nested_data__a_list__contains=2))\n", + "print(a_list_contains.inventory.hosts.keys())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can basically access any nested data by separating the elements in the path with two underscores `__`. Then you can use `__contains` to check if an element exists or if a string has a particular substring." + ] + } + ], + "metadata": { + "celltoolbar": "Edit Metadata", + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/howto/inventory/defaults.yaml b/docs/howto/inventory/defaults.yaml new file mode 100644 index 00000000..ddbc8dff --- /dev/null +++ b/docs/howto/inventory/defaults.yaml @@ -0,0 +1,4 @@ +--- +username: admin +data: + domain: acme.local diff --git a/docs/howto/inventory/groups.yaml b/docs/howto/inventory/groups.yaml new file mode 100644 index 00000000..0a9fc619 --- /dev/null +++ b/docs/howto/inventory/groups.yaml @@ -0,0 +1,21 @@ +--- +global: + data: + domain: global.local + asn: 1 + +eu: + data: + asn: 65100 + +bma: + groups: + - eu + - global + +cmh: + data: + asn: 65000 + vlans: + 100: frontend + 200: backend diff --git a/docs/howto/inventory/hosts.yaml b/docs/howto/inventory/hosts.yaml new file mode 100644 index 00000000..bc2a039b --- /dev/null +++ b/docs/howto/inventory/hosts.yaml @@ -0,0 +1,162 @@ +--- +host1.cmh: + hostname: 127.0.0.1 + port: 2201 + username: vagrant + password: vagrant + groups: + - cmh + platform: linux + data: + site: cmh + role: host + type: host + nested_data: + a_dict: + a: 1 + b: 2 + a_list: [1, 2] + a_string: "asdasd" + +host2.cmh: + hostname: 127.0.0.1 + port: 2202 + username: vagrant + password: vagrant + groups: + - cmh + platform: linux + data: + site: cmh + role: host + type: host + nested_data: + a_dict: + b: 2 + c: 3 + a_list: [1, 2] + a_string: "qwe" + +spine00.cmh: + hostname: 127.0.0.1 + username: vagrant + password: vagrant + port: 12444 + platform: eos + groups: + - cmh + data: + site: cmh + role: spine + type: network_device + +spine01.cmh: + hostname: 127.0.0.1 + username: vagrant + password: "" + port: 12204 + platform: junos + groups: + - cmh + data: + site: cmh + role: spine + type: network_device + +leaf00.cmh: + hostname: 127.0.0.1 + username: vagrant + password: vagrant + port: 12443 + groups: + - cmh + platform: eos + data: + site: cmh + role: leaf + type: network_device + asn: 65100 + +leaf01.cmh: + hostname: 127.0.0.1 + username: vagrant + password: "" + port: 12203 + groups: + - cmh + platform: junos + data: + site: cmh + role: leaf + type: network_device + asn: 65101 + +host1.bma: + groups: + - bma + platform: linux + data: + type: host + site: bma + role: host + +host2.bma: + groups: + - bma + platform: linux + data: + type: host + site: bma + role: host + +spine00.bma: + hostname: 127.0.0.1 + username: vagrant + password: vagrant + port: 12444 + groups: + - bma + platform: eos + data: + site: bma + role: spine + type: network_device + +spine01.bma: + hostname: 127.0.0.1 + username: vagrant + password: "" + port: 12204 + groups: + - bma + platform: junos + data: + type: network_device + site: bma + role: spine + +leaf00.bma: + hostname: 127.0.0.1 + username: vagrant + password: vagrant + port: 12443 + groups: + - bma + platform: eos + data: + type: network_device + site: bma + role: leaf + +leaf01.bma: + hostname: 127.0.0.1 + username: vagrant + password: wrong_password + port: 12203 + groups: + - bma + platform: junos + data: + type: network_device + site: bma + role: leaf diff --git a/nornir/core/__init__.py b/nornir/core/__init__.py index e0af8687..073ba7fd 100644 --- a/nornir/core/__init__.py +++ b/nornir/core/__init__.py @@ -194,6 +194,8 @@ def InitNornir(config_file="", dry_run=False, configure_logging=True, **kwargs): inv_class = conf.inventory.get_plugin() transform_function = conf.inventory.get_transform_function() - inv = inv_class(transform_function=transform_function, **conf.inventory.options) + inv = inv_class( + transform_function=transform_function, **conf.inventory.options + ).deserialize() return Nornir(inventory=inv, _config=conf) diff --git a/nornir/core/deserializer/__init__.py b/nornir/core/deserializer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nornir/core/deserializer/inventory.py b/nornir/core/deserializer/inventory.py new file mode 100644 index 00000000..3346b9dc --- /dev/null +++ b/nornir/core/deserializer/inventory.py @@ -0,0 +1,90 @@ +from typing import Any, Dict, List, Optional, Union + +from nornir.core import inventory + +from pydantic import BaseModel + +GroupsDict = None # DELETEME +HostsDict = None # DELETEME +VarsDict = None # DELETEME + + +class BaseAttributes(BaseModel): + hostname: Optional[str] = None + port: Optional[int] = None + username: Optional[str] = None + password: Optional[str] = None + platform: Optional[str] = None + + class Config: + ignore_extra = False + + +class ConnectionOptions(BaseAttributes): + extras: Dict[str, Any] = {} + + +class InventoryElement(BaseAttributes): + groups: List[str] = [] + data: Dict[str, Any] = {} + connection_options: Dict[str, ConnectionOptions] = {} + + @classmethod + def serialize(cls, e: Union[inventory.Host, inventory.Group]) -> "InventoryElement": + d = {} + for f in cls.__fields__: + d[f] = object.__getattribute__(e, f) + d["groups"] = list(d["groups"]) + return InventoryElement(**d) + + +class Defaults(BaseAttributes): + data: Dict[str, Any] = {} + connection_options: Dict[str, ConnectionOptions] = {} + + @classmethod + def serialize(cls, defaults: inventory.Defaults) -> "InventoryElement": + d = {} + for f in cls.__fields__: + d[f] = getattr(defaults, f) + return Defaults(**d) + + +class Inventory(BaseModel): + hosts: Dict[str, InventoryElement] + groups: Dict[str, InventoryElement] = {} + defaults: Defaults = {} + + @classmethod + def deserialize(cls, transform_function=None, *args, **kwargs): + deserialized = cls(*args, **kwargs) + + defaults = inventory.Defaults(**deserialized.defaults.dict()) + + hosts = inventory.Hosts() + for n, h in deserialized.hosts.items(): + h.groups = inventory.ParentGroups(h.groups) + hosts[n] = inventory.Host(defaults=defaults, name=n, **h.dict()) + + groups = inventory.Groups() + for n, g in deserialized.groups.items(): + g.groups = inventory.ParentGroups(g.groups) + groups[n] = inventory.Group(defaults=defaults, name=n, **g.dict()) + + return inventory.Inventory( + hosts=hosts, + groups=groups, + defaults=defaults, + transform_function=transform_function, + ) + + @classmethod + def serialize(cls, inv: inventory.Inventory): + hosts = {} + for n, h in inv.hosts.items(): + hosts[n] = InventoryElement.serialize(h) + groups = {} + for n, g in inv.groups.items(): + groups[n] = InventoryElement.serialize(g) + defaults = Defaults.serialize(inv.defaults) + return Inventory(hosts=hosts, groups=groups, defaults=defaults) diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 770a9290..eb5f41ba 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -1,47 +1,59 @@ -from collections import Sequence, UserList +from collections import UserList from typing import Any, Dict, List, Optional from nornir.core.configuration import Config from nornir.core.connections import Connections from nornir.core.exceptions import ConnectionAlreadyOpen, ConnectionNotOpen -from pydantic import BaseModel - GroupsDict = None # DELETEME HostsDict = None # DELETEME VarsDict = None # DELETEME -class BaseAttributes(BaseModel): - hostname: Optional[str] = None - port: Optional[int] = None - username: Optional[str] = None - password: Optional[str] = None - platform: Optional[str] = None +class BaseAttributes(object): + __slots__ = ("hostname", "port", "username", "password", "platform") + + def __init__( + self, + hostname: Optional[str] = None, + port: Optional[int] = None, + username: Optional[str] = None, + password: Optional[str] = None, + platform: Optional[str] = None, + ): + self.hostname = hostname + self.port = port + self.username = username + self.password = password + self.platform = platform + + def __recursive_slots__(self): + s = self.__slots__ + for b in self.__class__.__bases__: + if hasattr(b, "__recursive_slots__"): + s += b().__recursive_slots__() + elif hasattr(b, "__slots__"): + s += b.__slots__ + return s - class Config: - ignore_extra = False + def dict(self): + return {k: object.__getattribute__(self, k) for k in self.__recursive_slots__()} class ConnectionOptions(BaseAttributes): - extras: Dict[str, Any] = {} + __slots__ = ("extras",) + + def __init__(self, extras: Optional[Dict[str, Any]] = None, **kwargs): + self.extras = extras or {} + super().__init__(**kwargs) class ParentGroups(UserList): + __slots__ = "refs" + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.refs: Optional[List["Group"]] = [] - - @classmethod - def get_validators(cls): - yield cls.validate - - @classmethod - def validate(cls, v): - if not isinstance(v, Sequence): - raise ValueError(f"expected a list, got a {type(v)}") - - return ParentGroups(v) + self.refs: List["Group"] = kwargs.get("refs", []) def __contains__(self, value) -> bool: if isinstance(value, str): @@ -51,32 +63,49 @@ def __contains__(self, value) -> bool: class InventoryElement(BaseAttributes): - groups: ParentGroups = ParentGroups() - data: Dict[str, Any] = {} - connection_options: Dict[str, ConnectionOptions] = {} + __slots__ = ("groups", "data", "connection_options") + + def __init__( + self, + groups: Optional[ParentGroups] = None, + data: Optional[Dict[str, Any]] = None, + connection_options: Optional[Dict[str, ConnectionOptions]] = None, + **kwargs, + ): + self.groups = groups or ParentGroups() + self.data = data or {} + self.connection_options = connection_options or {} + super().__init__(**kwargs) class Defaults(BaseAttributes): - data: Dict[str, Any] = {} - connection_options: Dict[str, ConnectionOptions] = {} + __slots__ = ("data", "connection_options") + + def __init__( + self, + data: Optional[Dict[str, Any]] = None, + connection_options: Optional[Dict[str, ConnectionOptions]] = None, + **kwargs, + ): + self.data = data or {} + self.connection_options = connection_options or {} + super().__init__(**kwargs) class Host(InventoryElement): - name: str = "" - connections: Connections = Connections() - defaults: InventoryElement = InventoryElement() - config: Config = Config() - - class Config: - ignore_extra = False - - def dict(self, *args, **kwargs): - d = super().dict(*args, **kwargs) - d.pop("name") - d.pop("connections") - d.pop("defaults") - d.pop("config") - return d + __slots__ = ("name", "connections", "defaults", "config") + + def __init__( + self, + name: str, + defaults: Optional[InventoryElement] = None, + config: Optional[Config] = None, + **kwargs, + ): + self.name = name + self.defaults = defaults or Defaults() + self.connections: Connections = Connections() + super().__init__(**kwargs) def _resolve_data(self): processed = [] @@ -147,14 +176,14 @@ def __getitem__(self, item): def __getattribute__(self, name): if name not in ("hostname", "port", "username", "password", "platform"): return object.__getattribute__(self, name) - v = self.__values__[name] + v = object.__getattribute__(self, name) if v is None: for g in self.groups.refs: - r = g.__values__[name] + r = object.__getattribute__(self, name) if r is not None: return r - return self.defaults.__values__[name] + return object.__getattribute__(self.defaults, name) else: return v @@ -171,7 +200,7 @@ def __str__(self): return self.name def __repr__(self): - return "{}: {}".format(self.__class__.__name__, self.hostname or "") + return "{}: {}".format(self.__class__.__name__, self.name or "") def get(self, item, default=None): """ @@ -204,7 +233,7 @@ def get_connection_parameters( else: d = self._get_connection_options_recursively(connection) if d is not None: - return d + return ConnectionOptions(**d) else: d = { "hostname": self.hostname, @@ -341,52 +370,27 @@ class Groups(Dict[str, Group]): pass -class Inventory(BaseModel): - hosts: Hosts - groups: Groups = Groups() - defaults: Defaults = Defaults() - _config: Optional[Config] = None +class Inventory(object): + __slots__ = ("hosts", "groups", "defaults", "_config") def __init__( self, - hosts: Dict[str, Dict[str, Any]], - groups: Optional[Dict[str, Dict[str, Any]]] = None, - defaults: Optional[Dict[str, Any]] = None, + hosts: Hosts, + groups: Optional[Groups] = None, + defaults: Optional[Defaults] = None, transform_function=None, - *args, - **kwargs, ): - groups = groups or {} + self.hosts = hosts + self.groups = groups - if defaults is None: - defaults = Defaults() + for host in self.hosts.values(): + host.groups.refs = [self.groups[p] for p in host.groups] + for group in self.groups.values(): + group.groups.refs = [self.groups[p] for p in group.groups] - parsed_hosts = {} - for n, h in hosts.items(): - if isinstance(h, Host): - parsed_hosts[n] = h - else: - parsed_hosts[n] = Host(name=n, **h) - parsed_groups = {} - for n, g in groups.items(): - if isinstance(h, Host): - parsed_groups[n] = g - else: - parsed_groups[n] = Group(name=n, **g) - super().__init__( - hosts=parsed_hosts, groups=parsed_groups, defaults=defaults, *args, **kwargs - ) - - for n, h in parsed_hosts.items(): - h.defaults = self.defaults - for p in h.groups: - h.groups.refs.append(parsed_groups[p]) - for n, g in parsed_groups.items(): - for p in g.groups: - g.groups.refs.append(parsed_groups[p]) + self.defaults = defaults if transform_function: - h.defaults = self.defaults for h in self.hosts.values(): transform_function(h) diff --git a/nornir/plugins/inventory/netbox.py b/nornir/plugins/inventory/netbox.py index 60666e70..628098c6 100644 --- a/nornir/plugins/inventory/netbox.py +++ b/nornir/plugins/inventory/netbox.py @@ -1,6 +1,6 @@ import os -from nornir.core.inventory import Inventory +from nornir.core.deserializer.inventory import Inventory import requests diff --git a/tests/conftest.py b/tests/conftest.py index 23fed3fd..5dff2048 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,67 +1,67 @@ -import logging -import os -import subprocess +# import logging +# import os +# import subprocess -from nornir.core import InitNornir -from nornir.core.state import GlobalState +# from nornir.core import InitNornir +# from nornir.core.state import GlobalState -import pytest +# import pytest -global_data = GlobalState() +# global_data = GlobalState() -logging.basicConfig( - filename="tests.log", - filemode="w", - level=logging.DEBUG, - format="%(asctime)s - %(name)s - %(levelname)s - %(funcName)20s() - %(message)s", -) +# logging.basicConfig( +# filename="tests.log", +# filemode="w", +# level=logging.DEBUG, +# format="%(asctime)s - %(name)s - %(levelname)s - %(funcName)20s() - %(message)s", +# ) -@pytest.fixture(scope="session", autouse=True) -def containers(request): - """Start/Stop containers needed for the tests.""" +# @pytest.fixture(scope="session", autouse=True) +# def containers(request): +# """Start/Stop containers needed for the tests.""" - def fin(): - logging.info("Stopping containers") - subprocess.check_call( - ["./tests/inventory_data/containers.sh", "stop"], - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - ) +# def fin(): +# logging.info("Stopping containers") +# subprocess.check_call( +# ["./tests/inventory_data/containers.sh", "stop"], +# stderr=subprocess.PIPE, +# stdout=subprocess.PIPE, +# ) - request.addfinalizer(fin) +# request.addfinalizer(fin) - try: - fin() - except Exception: - pass - logging.info("Starting containers") - subprocess.check_call( - ["./tests/inventory_data/containers.sh", "start"], stdout=subprocess.PIPE - ) +# try: +# fin() +# except Exception: +# pass +# logging.info("Starting containers") +# subprocess.check_call( +# ["./tests/inventory_data/containers.sh", "start"], stdout=subprocess.PIPE +# ) -@pytest.fixture(scope="session", autouse=True) -def nornir(request): - """Initializes nornir""" - dir_path = os.path.dirname(os.path.realpath(__file__)) +# @pytest.fixture(scope="session", autouse=True) +# def nornir(request): +# """Initializes nornir""" +# dir_path = os.path.dirname(os.path.realpath(__file__)) - nornir = InitNornir( - inventory={ - "options": { - "host_file": "{}/inventory_data/hosts.yaml".format(dir_path), - "group_file": "{}/inventory_data/groups.yaml".format(dir_path), - "defaults_file": "{}/inventory_data/defaults.yaml".format(dir_path), - } - }, - dry_run=True, - ) - nornir.data = global_data - return nornir +# nornir = InitNornir( +# inventory={ +# "options": { +# "host_file": "{}/inventory_data/hosts.yaml".format(dir_path), +# "group_file": "{}/inventory_data/groups.yaml".format(dir_path), +# "defaults_file": "{}/inventory_data/defaults.yaml".format(dir_path), +# } +# }, +# dry_run=True, +# ) +# nornir.data = global_data +# return nornir -@pytest.fixture(scope="function", autouse=True) -def reset_failed_hosts(): - global_data.reset_failed_hosts() +# @pytest.fixture(scope="function", autouse=True) +# def reset_failed_hosts(): +# global_data.reset_failed_hosts() diff --git a/tests/core/test_InitNornir.py b/tests/core/test_InitNornir.py index 053aa86a..c10f31a8 100644 --- a/tests/core/test_InitNornir.py +++ b/tests/core/test_InitNornir.py @@ -2,7 +2,7 @@ from nornir.core import InitNornir -from nornir.core.inventory import Inventory +from nornir.core.deserializer.inventory import Inventory dir_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "test_InitNornir") @@ -14,7 +14,7 @@ def transform_func(host): def StringInventory(**kwargs): inv_dict = {"hosts": {"host1": {}, "host2": {}}, "groups": {}, "defaults": {}} - return Inventory(**inv_dict, **kwargs) + return Inventory.deserialize(**inv_dict, **kwargs) class Test(object): diff --git a/tests/core/test_filter.py b/tests/core/test_filter.py index 33120d72..8103bca7 100644 --- a/tests/core/test_filter.py +++ b/tests/core/test_filter.py @@ -104,7 +104,7 @@ def test_filtering_by_callable_has_parent_group(self): f = F(has_parent_group="parent_group") filtered = sorted(list((inventory.filter(f).hosts.keys()))) - assert filtered == ["dev1.group_1", "dev2.group_1"] + assert filtered == ["dev1.group_1", "dev2.group_1", "dev4.group_2"] def test_filtering_by_attribute_name(self): f = F(name="dev1.group_1") diff --git a/tests/core/test_inventory.py b/tests/core/test_inventory.py index 82c9453b..2b5ba88b 100644 --- a/tests/core/test_inventory.py +++ b/tests/core/test_inventory.py @@ -1,6 +1,7 @@ import os -from nornir.core.inventory import Group, Host, Inventory +from nornir.core import inventory +from nornir.core.deserializer import inventory as deserializer from pydantic import ValidationError @@ -22,7 +23,7 @@ class Test(object): def test_host(self): - h = Host(name="host1", hostname="host1") + h = inventory.Host(name="host1", hostname="host1") assert h.hostname == "host1" assert h.port is None assert h.username is None @@ -31,7 +32,7 @@ def test_host(self): assert h.data == {} data = {"asn": 65100, "router_id": "1.1.1.1"} - h = Host( + h = inventory.Host( name="host2", hostname="host2", username="user", @@ -48,30 +49,32 @@ def test_host(self): assert h.data == data def test_inventory(self): - g1 = Group(name="g1") - g2 = Group(name="g2", groups=["g1"]) - h1 = Host(name="h1", groups=["g1", "g2"]) - h2 = Host(name="h2") + g1 = inventory.Group(name="g1") + g2 = inventory.Group(name="g2", groups=inventory.ParentGroups(["g1"])) + h1 = inventory.Host(name="h1", groups=inventory.ParentGroups(["g1", "g2"])) + h2 = inventory.Host(name="h2") hosts = {"h1": h1, "h2": h2} groups = {"g1": g1, "g2": g2} - inventory = Inventory(hosts=hosts, groups=groups) - assert "h1" in inventory.hosts - assert "h2" in inventory.hosts - assert "g1" in inventory.groups - assert "g2" in inventory.groups - assert inventory.groups["g1"] in inventory.hosts["h1"].groups - assert inventory.groups["g1"] in inventory.groups["g2"].groups + inv = inventory.Inventory(hosts=hosts, groups=groups) + assert "h1" in inv.hosts + assert "h2" in inv.hosts + assert "g1" in inv.groups + assert "g2" in inv.groups + assert inv.groups["g1"] in inv.hosts["h1"].groups + assert inv.groups["g1"] in inv.groups["g2"].groups def test_inventory_deserializer_wrong(self): with pytest.raises(ValidationError): - Inventory(**{"hosts": {"wrong": {"host": "should_be_hostname"}}}) + deserializer.Inventory.deserialize( + **{"hosts": {"wrong": {"host": "should_be_hostname"}}} + ) def test_inventory_deserializer(self): - inv = Inventory(**inv_dict) + inv = deserializer.Inventory.deserialize(**inv_dict) assert inv.groups["group_1"] in inv.hosts["dev1.group_1"].groups def test_filtering(self): - inv = Inventory(**inv_dict) + inv = deserializer.Inventory.deserialize(**inv_dict) unfiltered = sorted(list(inv.hosts.keys())) assert unfiltered == [ "dev1.group_1", @@ -92,7 +95,7 @@ def test_filtering(self): assert www_site1 == ["dev1.group_1"] def test_filtering_func(self): - inv = Inventory(**inv_dict) + inv = deserializer.Inventory.deserialize(**inv_dict) long_names = sorted( list(inv.filter(filter_func=lambda x: len(x["my_var"]) > 20).hosts.keys()) ) @@ -107,14 +110,14 @@ def longer_than(dev, length): assert long_names == ["dev1.group_1", "dev4.group_2"] def test_filter_unique_keys(self): - inv = Inventory(**inv_dict) + inv = deserializer.Inventory.deserialize(**inv_dict) filtered = sorted(list(inv.filter(www_server="nginx").hosts.keys())) assert filtered == ["dev1.group_1"] def test_var_resolution(self): - inv = Inventory(**inv_dict) - # assert inv.hosts["dev1.group_1"]["my_var"] == "comes_from_dev1.group_1" - # assert inv.hosts["dev2.group_1"]["my_var"] == "comes_from_group_1" + inv = deserializer.Inventory.deserialize(**inv_dict) + assert inv.hosts["dev1.group_1"]["my_var"] == "comes_from_dev1.group_1" + assert inv.hosts["dev2.group_1"]["my_var"] == "comes_from_group_1" assert inv.hosts["dev3.group_2"]["my_var"] == "comes_from_defaults" assert inv.hosts["dev4.group_2"]["my_var"] == "comes_from_dev4.group_2" @@ -129,19 +132,20 @@ def test_var_resolution(self): assert inv.hosts["dev2.group_1"].password == "docker" def test_has_parents(self): - inv = Inventory(**inv_dict) + inv = deserializer.Inventory.deserialize(**inv_dict) assert inv.hosts["dev1.group_1"].has_parent_group(inv.groups["group_1"]) assert not inv.hosts["dev1.group_1"].has_parent_group(inv.groups["group_2"]) assert inv.hosts["dev1.group_1"].has_parent_group("group_1") assert not inv.hosts["dev1.group_1"].has_parent_group("group_2") def test_to_dict(self): - inv = Inventory(**inv_dict).dict() + inv = deserializer.Inventory.deserialize(**inv_dict) + inv_serialized = deserializer.Inventory.serialize(inv).dict() for k, v in inv_dict.items(): - assert v == inv[k] + assert v == inv_serialized[k] def test_get_connection_parameters(self): - inv = Inventory(**inv_dict) + inv = deserializer.Inventory.deserialize(**inv_dict) p1 = inv.hosts["dev1.group_1"].get_connection_parameters("dummy") assert p1.dict() == { "port": None, @@ -180,7 +184,7 @@ def test_get_connection_parameters(self): } def test_defaults(self): - inv = Inventory(**inv_dict) + inv = deserializer.Inventory.deserialize(**inv_dict) inv.defaults.password = "asd" assert inv.defaults.password == "asd" assert inv.hosts["dev2.group_1"].password == "asd" diff --git a/tests/inventory_data/hosts.yaml b/tests/inventory_data/hosts.yaml index 2cb7e224..905ed2f0 100644 --- a/tests/inventory_data/hosts.yaml +++ b/tests/inventory_data/hosts.yaml @@ -80,4 +80,5 @@ dev4.group_2: role: db groups: - group_2 + - parent_group connection_options: {} diff --git a/tests/plugins/inventory/test_netbox.py b/tests/plugins/inventory/test_netbox.py index 499e7c71..3e39bfc5 100644 --- a/tests/plugins/inventory/test_netbox.py +++ b/tests/plugins/inventory/test_netbox.py @@ -17,7 +17,7 @@ def get_inv(requests_mock, case, **kwargs): json=json.load(f), headers={"Content-type": "application/json"}, ) - return netbox.NBInventory(**kwargs) + return netbox.NBInventory.deserialize(**kwargs) def transform_function(host): From 620d44648f7e8c5f59a4216f2499d14940c741e2 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Tue, 4 Sep 2018 13:36:33 +0200 Subject: [PATCH 054/109] progress --- nornir/core/__init__.py | 6 +- nornir/core/inventory.py | 9 ++- nornir/plugins/inventory/nsot.py | 18 ++--- nornir/plugins/inventory/simple.py | 2 +- tests/conftest.py | 102 ++++++++++++------------- tests/core/test_InitNornir.py | 7 +- tests/core/test_filter.py | 87 ++++++++++----------- tests/plugins/inventory/test_netbox.py | 5 +- tests/plugins/inventory/test_nsot.py | 2 +- 9 files changed, 116 insertions(+), 122 deletions(-) diff --git a/nornir/core/__init__.py b/nornir/core/__init__.py index 073ba7fd..3be4e127 100644 --- a/nornir/core/__init__.py +++ b/nornir/core/__init__.py @@ -194,8 +194,8 @@ def InitNornir(config_file="", dry_run=False, configure_logging=True, **kwargs): inv_class = conf.inventory.get_plugin() transform_function = conf.inventory.get_transform_function() - inv = inv_class( - transform_function=transform_function, **conf.inventory.options - ).deserialize() + inv = inv_class.deserialize( + transform_function=transform_function, config=conf, **conf.inventory.options + ) return Nornir(inventory=inv, _config=conf) diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index eb5f41ba..204ff75a 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -63,18 +63,20 @@ def __contains__(self, value) -> bool: class InventoryElement(BaseAttributes): - __slots__ = ("groups", "data", "connection_options") + __slots__ = ("groups", "data", "connection_options", "config") def __init__( self, groups: Optional[ParentGroups] = None, data: Optional[Dict[str, Any]] = None, connection_options: Optional[Dict[str, ConnectionOptions]] = None, + config: Optional[Config] = None, **kwargs, ): self.groups = groups or ParentGroups() self.data = data or {} self.connection_options = connection_options or {} + self.config = config or Config() super().__init__(**kwargs) @@ -378,6 +380,7 @@ def __init__( hosts: Hosts, groups: Optional[Groups] = None, defaults: Optional[Defaults] = None, + config: Optional[Config] = None, transform_function=None, ): self.hosts = hosts @@ -394,6 +397,8 @@ def __init__( for h in self.hosts.values(): transform_function(h) + self.config = config or Config() + def filter(self, filter_obj=None, filter_func=None, *args, **kwargs): filter_func = filter_obj or filter_func if filter_func: @@ -417,4 +422,4 @@ def config(self): def config(self, value): self._config = value for host in self.hosts.values(): - host._config = value + host.config = value diff --git a/nornir/plugins/inventory/nsot.py b/nornir/plugins/inventory/nsot.py index 254faa67..a55d0a5c 100644 --- a/nornir/plugins/inventory/nsot.py +++ b/nornir/plugins/inventory/nsot.py @@ -1,9 +1,9 @@ import os -from typing import Any, List +from typing import Any -import requests +from nornir.core.deserializer.inventory import Inventory, InventoryElement -from nornir.core.inventory import Inventory, VarsDict, HostsDict, InventoryElement +import requests class NSOTInventory(Inventory): @@ -58,13 +58,9 @@ def __init__( ) headers = {nsot_auth_header: nsot_email} - devices: List[VarsDict] = requests.get( - "{}/devices".format(nsot_url), headers=headers - ).json() - sites: List[VarsDict] = requests.get( - "{}/sites".format(nsot_url), headers=headers - ).json() - interfaces: List[VarsDict] = requests.get( + devices = requests.get("{}/devices".format(nsot_url), headers=headers).json() + sites = requests.get("{}/sites".format(nsot_url), headers=headers).json() + interfaces = requests.get( "{}/interfaces".format(nsot_url), headers=headers ).json() @@ -90,5 +86,5 @@ def __init__( devices[i["device"] - 1]["data"]["interfaces"][i["name"]] = i # Finally the inventory expects a dict of hosts where the key is the hostname - hosts: HostsDict = {d["hostname"]: d for d in devices} + hosts = {d["hostname"]: d for d in devices} return super().__init__(hosts=hosts, groups={}, defaults={}, *args, **kwargs) diff --git a/nornir/plugins/inventory/simple.py b/nornir/plugins/inventory/simple.py index f4547e39..b67fd1de 100644 --- a/nornir/plugins/inventory/simple.py +++ b/nornir/plugins/inventory/simple.py @@ -1,7 +1,7 @@ import logging import os -from nornir.core.inventory import Inventory +from nornir.core.deserializer.inventory import Inventory import ruamel.yaml diff --git a/tests/conftest.py b/tests/conftest.py index 5dff2048..23fed3fd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,67 +1,67 @@ -# import logging -# import os -# import subprocess +import logging +import os +import subprocess -# from nornir.core import InitNornir -# from nornir.core.state import GlobalState +from nornir.core import InitNornir +from nornir.core.state import GlobalState -# import pytest +import pytest -# global_data = GlobalState() +global_data = GlobalState() -# logging.basicConfig( -# filename="tests.log", -# filemode="w", -# level=logging.DEBUG, -# format="%(asctime)s - %(name)s - %(levelname)s - %(funcName)20s() - %(message)s", -# ) +logging.basicConfig( + filename="tests.log", + filemode="w", + level=logging.DEBUG, + format="%(asctime)s - %(name)s - %(levelname)s - %(funcName)20s() - %(message)s", +) -# @pytest.fixture(scope="session", autouse=True) -# def containers(request): -# """Start/Stop containers needed for the tests.""" +@pytest.fixture(scope="session", autouse=True) +def containers(request): + """Start/Stop containers needed for the tests.""" -# def fin(): -# logging.info("Stopping containers") -# subprocess.check_call( -# ["./tests/inventory_data/containers.sh", "stop"], -# stderr=subprocess.PIPE, -# stdout=subprocess.PIPE, -# ) + def fin(): + logging.info("Stopping containers") + subprocess.check_call( + ["./tests/inventory_data/containers.sh", "stop"], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) -# request.addfinalizer(fin) + request.addfinalizer(fin) -# try: -# fin() -# except Exception: -# pass -# logging.info("Starting containers") -# subprocess.check_call( -# ["./tests/inventory_data/containers.sh", "start"], stdout=subprocess.PIPE -# ) + try: + fin() + except Exception: + pass + logging.info("Starting containers") + subprocess.check_call( + ["./tests/inventory_data/containers.sh", "start"], stdout=subprocess.PIPE + ) -# @pytest.fixture(scope="session", autouse=True) -# def nornir(request): -# """Initializes nornir""" -# dir_path = os.path.dirname(os.path.realpath(__file__)) +@pytest.fixture(scope="session", autouse=True) +def nornir(request): + """Initializes nornir""" + dir_path = os.path.dirname(os.path.realpath(__file__)) -# nornir = InitNornir( -# inventory={ -# "options": { -# "host_file": "{}/inventory_data/hosts.yaml".format(dir_path), -# "group_file": "{}/inventory_data/groups.yaml".format(dir_path), -# "defaults_file": "{}/inventory_data/defaults.yaml".format(dir_path), -# } -# }, -# dry_run=True, -# ) -# nornir.data = global_data -# return nornir + nornir = InitNornir( + inventory={ + "options": { + "host_file": "{}/inventory_data/hosts.yaml".format(dir_path), + "group_file": "{}/inventory_data/groups.yaml".format(dir_path), + "defaults_file": "{}/inventory_data/defaults.yaml".format(dir_path), + } + }, + dry_run=True, + ) + nornir.data = global_data + return nornir -# @pytest.fixture(scope="function", autouse=True) -# def reset_failed_hosts(): -# global_data.reset_failed_hosts() +@pytest.fixture(scope="function", autouse=True) +def reset_failed_hosts(): + global_data.reset_failed_hosts() diff --git a/tests/core/test_InitNornir.py b/tests/core/test_InitNornir.py index c10f31a8..cce21086 100644 --- a/tests/core/test_InitNornir.py +++ b/tests/core/test_InitNornir.py @@ -12,9 +12,10 @@ def transform_func(host): host["processed_by_transform_function"] = True -def StringInventory(**kwargs): - inv_dict = {"hosts": {"host1": {}, "host2": {}}, "groups": {}, "defaults": {}} - return Inventory.deserialize(**inv_dict, **kwargs) +class StringInventory(Inventory): + def __init__(self, **kwargs): + inv_dict = {"hosts": {"host1": {}, "host2": {}}, "groups": {}, "defaults": {}} + super().__init__(**inv_dict, **kwargs) class Test(object): diff --git a/tests/core/test_filter.py b/tests/core/test_filter.py index 8103bca7..ad58b93c 100644 --- a/tests/core/test_filter.py +++ b/tests/core/test_filter.py @@ -1,131 +1,122 @@ -import os - from nornir.core.filter import F -from nornir.plugins.inventory.simple import SimpleInventory - -dir_path = os.path.dirname(os.path.realpath(__file__)) -inventory = SimpleInventory( - "{}/../inventory_data/hosts.yaml".format(dir_path), - "{}/../inventory_data/groups.yaml".format(dir_path), -) class Test(object): - def test_simple(self): + def test_simple(self, nornir): f = F(site="site1") - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev1.group_1", "dev2.group_1"] - def test_and(self): + def test_and(self, nornir): f = F(site="site1") & F(role="www") - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev1.group_1"] - def test_or(self): + def test_or(self, nornir): f = F(site="site1") | F(role="www") - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev1.group_1", "dev2.group_1", "dev3.group_2"] - def test_combined(self): + def test_combined(self, nornir): f = F(site="site2") | (F(role="www") & F(my_var="comes_from_dev1.group_1")) - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev1.group_1", "dev3.group_2", "dev4.group_2"] f = (F(site="site2") | F(role="www")) & F(my_var="comes_from_dev1.group_1") - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev1.group_1"] - def test_contains(self): + def test_contains(self, nornir): f = F(groups__contains="group_1") - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev1.group_1", "dev2.group_1"] - def test_negate(self): + def test_negate(self, nornir): f = ~F(groups__contains="group_1") - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev3.group_2", "dev4.group_2"] - def test_negate_and_second_negate(self): + def test_negate_and_second_negate(self, nornir): f = F(site="site1") & ~F(role="www") - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev2.group_1"] - def test_negate_or_both_negate(self): + def test_negate_or_both_negate(self, nornir): f = ~F(site="site1") | ~F(role="www") - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev2.group_1", "dev3.group_2", "dev4.group_2"] - def test_nested_data_a_string(self): + def test_nested_data_a_string(self, nornir): f = F(nested_data__a_string="asdasd") - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev1.group_1"] - def test_nested_data_a_string_contains(self): + def test_nested_data_a_string_contains(self, nornir): f = F(nested_data__a_string__contains="asd") - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev1.group_1"] - def test_nested_data_a_dict_contains(self): + def test_nested_data_a_dict_contains(self, nornir): f = F(nested_data__a_dict__contains="a") - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev1.group_1"] - def test_nested_data_a_dict_element(self): + def test_nested_data_a_dict_element(self, nornir): f = F(nested_data__a_dict__a=1) - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev1.group_1"] - def test_nested_data_a_dict_doesnt_contain(self): + def test_nested_data_a_dict_doesnt_contain(self, nornir): f = ~F(nested_data__a_dict__contains="a") - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev2.group_1", "dev3.group_2", "dev4.group_2"] - def test_nested_data_a_list_contains(self): + def test_nested_data_a_list_contains(self, nornir): f = F(nested_data__a_list__contains=2) - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev1.group_1", "dev2.group_1"] - def test_filtering_by_callable_has_parent_group(self): + def test_filtering_by_callable_has_parent_group(self, nornir): f = F(has_parent_group="parent_group") - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev1.group_1", "dev2.group_1", "dev4.group_2"] - def test_filtering_by_attribute_name(self): + def test_filtering_by_attribute_name(self, nornir): f = F(name="dev1.group_1") - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev1.group_1"] - def test_filtering_string_in_list(self): + def test_filtering_string_in_list(self, nornir): f = F(platform__in=["linux", "mock"]) - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev3.group_2", "dev4.group_2"] - def test_filtering_list_any(self): + def test_filtering_list_any(self, nornir): f = F(nested_data__a_list__any=[1, 3]) - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev1.group_1", "dev2.group_1"] - def test_filtering_list_all(self): + def test_filtering_list_all(self, nornir): f = F(nested_data__a_list__all=[1, 2]) - filtered = sorted(list((inventory.filter(f).hosts.keys()))) + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev1.group_1"] diff --git a/tests/plugins/inventory/test_netbox.py b/tests/plugins/inventory/test_netbox.py index 3e39bfc5..33158a55 100644 --- a/tests/plugins/inventory/test_netbox.py +++ b/tests/plugins/inventory/test_netbox.py @@ -1,6 +1,7 @@ import json import os +from nornir.core.deserializer.inventory import Inventory from nornir.plugins.inventory import netbox # We need import below to load fixtures @@ -32,7 +33,7 @@ def test_inventory(self, requests_mock): # f.write(InventorySerializer.serialize(inv).json()) with open("{}/{}/expected.json".format(BASE_PATH, "2.3.5"), "r") as f: expected = json.load(f) - assert expected == inv.dict() + assert expected == Inventory.serialize(inv).dict() def test_transform_function(self, requests_mock): inv = get_inv(requests_mock, "2.3.5", transform_function=transform_function) @@ -44,4 +45,4 @@ def test_transform_function(self, requests_mock): "{}/{}/expected_transform_function.json".format(BASE_PATH, "2.3.5"), "r" ) as f: expected = json.load(f) - assert expected == inv.dict() + assert expected == Inventory.serialize(inv).dict() diff --git a/tests/plugins/inventory/test_nsot.py b/tests/plugins/inventory/test_nsot.py index 5c91261c..0e6e7f26 100644 --- a/tests/plugins/inventory/test_nsot.py +++ b/tests/plugins/inventory/test_nsot.py @@ -18,7 +18,7 @@ def get_inv(requests_mock, case, **kwargs): json=json.load(f), headers={"Content-type": "application/json"}, ) - return nsot.NSOTInventory(**kwargs) + return nsot.NSOTInventory.deserialize(**kwargs) def transform_function(host): From 61050526a65b9de4788689770d54859672a031cf Mon Sep 17 00:00:00 2001 From: David Barroso Date: Tue, 4 Sep 2018 14:31:57 +0200 Subject: [PATCH 055/109] fix tests --- docs/howto/handling_connections.ipynb | 26 +- docs/howto/inventory.ipynb | 649 +++++++++++++++--- docs/howto/transforming_inventory_data.ipynb | 72 +- .../writing_a_custom_inventory_plugin.ipynb | 14 +- .../my_inventory.py | 4 +- nornir/core/inventory.py | 5 +- setup.cfg | 2 +- 7 files changed, 573 insertions(+), 199 deletions(-) diff --git a/docs/howto/handling_connections.ipynb b/docs/howto/handling_connections.ipynb index 166bf521..d8912a2f 100644 --- a/docs/howto/handling_connections.ipynb +++ b/docs/howto/handling_connections.ipynb @@ -82,17 +82,21 @@ "text": [ "\u001b[1m\u001b[36mtask_manages_connection_manually************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[34m* rtr00 ** changed : False *****************************************************\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[31mvvvv task_manages_connection_manually ** changed : False vvvvvvvvvvvvvvvvvvvvvvv ERROR\u001b[0m\n", - "\u001b[0mTraceback (most recent call last):\n", - " File \"/Users/dbarroso/workspace/nornir/nornir/core/task.py\", line 63, in start\n", - " r = self.task(self, **self.params)\n", - " File \"\", line 5, in task_manages_connection_manually\n", - " getters=[\"facts\"]\n", - " File \"/Users/dbarroso/workspace/nornir/nornir/core/task.py\", line 100, in run\n", - " raise Exception(msg)\n", - "Exception: ('You have to call this after setting host and nornir attributes. ', 'You probably called this from outside a nested task')\n", - "\u001b[0m\n", - "\u001b[0m\u001b[1m\u001b[31m^^^^ END task_manages_connection_manually ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32mvvvv task_manages_connection_manually ** changed : False vvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m---- napalm_get ** changed : False --------------------------------------------- INFO\u001b[0m\n", + "\u001b[0m{\u001b[0m \u001b[0m'facts'\u001b[0m: \u001b[0m{\u001b[0m \u001b[0m'fqdn'\u001b[0m: \u001b[0m'localhost'\u001b[0m,\n", + " \u001b[0m'hostname'\u001b[0m: \u001b[0m'localhost'\u001b[0m,\n", + " \u001b[0m'interface_list'\u001b[0m: \u001b[0m[\u001b[0m \u001b[0m\u001b[0m'Ethernet1'\u001b[0m,\n", + " \u001b[0m'Ethernet2'\u001b[0m,\n", + " \u001b[0m'Ethernet3'\u001b[0m,\n", + " \u001b[0m'Ethernet4'\u001b[0m,\n", + " \u001b[0m'Management1'\u001b[0m]\u001b[0m,\n", + " \u001b[0m'model'\u001b[0m: \u001b[0m'vEOS'\u001b[0m,\n", + " \u001b[0m'os_version'\u001b[0m: \u001b[0m'4.15.5M-3054042.4155M'\u001b[0m,\n", + " \u001b[0m'serial_number'\u001b[0m: \u001b[0m''\u001b[0m,\n", + " \u001b[0m'uptime'\u001b[0m: \u001b[0m'...'\u001b[0m,\n", + " \u001b[0m'vendor'\u001b[0m: \u001b[0m'Arista'\u001b[0m}\u001b[0m}\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m^^^^ END task_manages_connection_manually ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", "\u001b[0m" ] } diff --git a/docs/howto/inventory.ipynb b/docs/howto/inventory.ipynb index 241dc9b7..aec08a41 100644 --- a/docs/howto/inventory.ipynb +++ b/docs/howto/inventory.ipynb @@ -18,9 +18,9 @@ "source": [ "## Inventory\n", "\n", - "The Inventory is arguably the most important piece of nornir. Let's see how it works. To begin with the [inventory](../../ref/api/inventory.rst#nornir.core.inventory.Inventory) is comprised of [hosts](../../ref/api/inventory.rst#nornir.core.inventory.Host) and [groups](../../ref/api/inventory.rst#nornir.core.inventory.Group).\n", + "The Inventory is arguably the most important piece of nornir. Let's see how it works. To begin with the [inventory](../../ref/api/inventory.rst#nornir.core.inventory.Inventory) is comprised of [hosts](../../ref/api/inventory.rst#nornir.core.inventory.Host), [groups](../../ref/api/inventory.rst#nornir.core.inventory.Group) and [defaults](../../ref/api/inventory.rst#nornir.core.inventory.Defaults).\n", "\n", - "In this tutorial we are using the [SimpleInventory](../../plugins/inventory/simple.rst#nornir.plugins.inventory.simple.SimpleInventory) plugin. This inventory plugin stores all the relevant data in two files. Let's start by checking them:" + "In this tutorial we are using the [SimpleInventory](../../plugins/inventory/simple.rst#nornir.plugins.inventory.simple.SimpleInventory) plugin. This inventory plugin stores all the relevant data in three files. Let's start by checking them:" ] }, { @@ -288,15 +288,122 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The hosts file is basically a map where the outermost key is the hostname and then any arbitrary `` pair you want inside. Some keys like hostname, username or password have special meaning, you can investigate the [hosts](../../ref/api/inventory.rst#nornir.core.inventory.Host) class for details on those. In addition, the `groups` key is a list of groups you can inherite data from. We will inspect soon how the inheritance model works.\n", - "\n", - "Now, let's look at the groups file:" + "The hosts file is basically a map where the outermost key is the name of the host and then an `InventoryElement` object. You can see the schema of the object by executing:" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"title\": \"InventoryElement\",\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"hostname\": {\n", + " \"title\": \"Hostname\",\n", + " \"required\": false,\n", + " \"type\": \"str\"\n", + " },\n", + " \"port\": {\n", + " \"title\": \"Port\",\n", + " \"required\": false,\n", + " \"type\": \"int\"\n", + " },\n", + " \"username\": {\n", + " \"title\": \"Username\",\n", + " \"required\": false,\n", + " \"type\": \"str\"\n", + " },\n", + " \"password\": {\n", + " \"title\": \"Password\",\n", + " \"required\": false,\n", + " \"type\": \"str\"\n", + " },\n", + " \"platform\": {\n", + " \"title\": \"Platform\",\n", + " \"required\": false,\n", + " \"type\": \"str\"\n", + " },\n", + " \"groups\": {\n", + " \"title\": \"Groups\",\n", + " \"required\": false,\n", + " \"default\": [],\n", + " \"type\": \"list\",\n", + " \"item_type\": \"str\"\n", + " },\n", + " \"data\": {\n", + " \"title\": \"Data\",\n", + " \"required\": false,\n", + " \"default\": {},\n", + " \"type\": \"mapping\",\n", + " \"item_type\": \"any\",\n", + " \"key_type\": \"str\"\n", + " },\n", + " \"connection_options\": {\n", + " \"title\": \"Connection_Options\",\n", + " \"required\": false,\n", + " \"default\": {},\n", + " \"type\": \"mapping\",\n", + " \"item_type\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"hostname\": {\n", + " \"title\": \"Hostname\",\n", + " \"required\": false,\n", + " \"type\": \"str\"\n", + " },\n", + " \"port\": {\n", + " \"title\": \"Port\",\n", + " \"required\": false,\n", + " \"type\": \"int\"\n", + " },\n", + " \"username\": {\n", + " \"title\": \"Username\",\n", + " \"required\": false,\n", + " \"type\": \"str\"\n", + " },\n", + " \"password\": {\n", + " \"title\": \"Password\",\n", + " \"required\": false,\n", + " \"type\": \"str\"\n", + " },\n", + " \"platform\": {\n", + " \"title\": \"Platform\",\n", + " \"required\": false,\n", + " \"type\": \"str\"\n", + " },\n", + " \"extras\": {\n", + " \"title\": \"Extras\",\n", + " \"required\": false,\n", + " \"default\": {},\n", + " \"type\": \"mapping\",\n", + " \"item_type\": \"any\",\n", + " \"key_type\": \"str\"\n", + " }\n", + " }\n", + " },\n", + " \"key_type\": \"str\"\n", + " }\n", + " }\n", + "}\n" + ] + } + ], + "source": [ + "from nornir.core.deserializer.inventory import InventoryElement\n", + "import json\n", + "print(json.dumps(InventoryElement.schema(), indent=4))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, "outputs": [ { "data": { @@ -403,7 +510,7 @@ "" ] }, - "execution_count": 3, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -413,9 +520,16 @@ "%highlight_file inventory/groups.yaml" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The groups file works the same way as the hosts file." + ] + }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -506,7 +620,7 @@ "" ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -517,54 +631,10 @@ ] }, { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'properties': {'defaults': {'default': ,\n", - " 'properties': {...},\n", - " 'required': False,\n", - " 'title': 'Defaults',\n", - " 'type': 'object'},\n", - " 'groups': {'default': {},\n", - " 'required': False,\n", - " 'title': 'Groups',\n", - " 'type': 'Groups'},\n", - " 'hosts': {'required': True, 'title': 'Hosts', 'type': 'Hosts'}},\n", - " 'title': 'Inventory',\n", - " 'type': 'object'}\n" - ] - } - ], - "source": [ - "from nornir.core.inventory import Inventory\n", - "import pprint\n", - "pprint.pprint(Inventory.schema(), depth=3)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "ename": "Exception", - "evalue": "666", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mException\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m666\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mException\u001b[0m: 666" - ] - } - ], "source": [ - "raise Exception(666)" + "Finally, the defaults file has the same schema as the ``InventoryElement`` we described before. We will see how the data in the groups and defaults file is used later on in this tutorial." ] }, { @@ -580,7 +650,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -593,9 +663,7 @@ " \"defaults_file\": \"inventory/defaults.yaml\",\n", " }\n", " }\n", - ")\n", - "\n", - "nr.inventory" + ")" ] }, { @@ -607,27 +675,74 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'host1.cmh': Host: host1.cmh,\n", + " 'host2.cmh': Host: host2.cmh,\n", + " 'spine00.cmh': Host: spine00.cmh,\n", + " 'spine01.cmh': Host: spine01.cmh,\n", + " 'leaf00.cmh': Host: leaf00.cmh,\n", + " 'leaf01.cmh': Host: leaf01.cmh,\n", + " 'host1.bma': Host: host1.bma,\n", + " 'host2.bma': Host: host2.bma,\n", + " 'spine00.bma': Host: spine00.bma,\n", + " 'spine01.bma': Host: spine01.bma,\n", + " 'leaf00.bma': Host: leaf00.bma,\n", + " 'leaf01.bma': Host: leaf01.bma}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "nr.inventory.hosts" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'global': Group: global,\n", + " 'eu': Group: eu,\n", + " 'bma': Group: bma,\n", + " 'cmh': Group: cmh}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "nr.inventory.groups" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Host: leaf01.bma" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "nr.inventory.hosts[\"leaf01.bma\"]" ] @@ -641,9 +756,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['type', 'site', 'role', 'asn', 'domain'])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "host = nr.inventory.hosts[\"leaf01.bma\"]\n", "host.keys()" @@ -651,11 +777,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'bma'" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "host[\"site\"]" ] @@ -671,9 +808,119 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
 1 ---\n",
+       " 2 global:\n",
+       " 3     data:\n",
+       " 4         domain: global.local\n",
+       " 5         asn: 1\n",
+       " 6 \n",
+       " 7 eu:\n",
+       " 8     data:\n",
+       " 9         asn: 65100\n",
+       "10 \n",
+       "11 bma:\n",
+       "12     groups:\n",
+       "13         - eu\n",
+       "14         - global\n",
+       "15 \n",
+       "16 cmh:\n",
+       "17     data:\n",
+       "18         asn: 65000\n",
+       "19         vlans:\n",
+       "20           100: frontend\n",
+       "21           200: backend\n",
+       "
\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# groups file\n", "%highlight_file inventory/groups.yaml" @@ -690,9 +937,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'acme.local'" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "leaf01_bma = nr.inventory.hosts[\"leaf01.bma\"]\n", "leaf01_bma[\"domain\"] # comes from the group `global`" @@ -700,9 +958,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "65100" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "leaf01_bma[\"asn\"] # comes from group `eu`" ] @@ -711,14 +980,25 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The group `defaults` is special. This group contains data that will be returned if neither the host nor the parents have a specific value for it." + "Finally, the `defaults` file contains data that will be returned if neither the host nor the parents have a specific value for it." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'acme.local'" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "leaf01_cmh = nr.inventory.hosts[\"leaf01.cmh\"]\n", "leaf01_cmh[\"domain\"] # comes from defaults" @@ -733,9 +1013,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Couldn't find key: 'non_existent'\n" + ] + } + ], "source": [ "try:\n", " leaf01_cmh[\"non_existent\"]\n", @@ -752,9 +1040,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Couldn't find key: 'domain'\n" + ] + } + ], "source": [ "try:\n", " leaf01_cmh.data[\"domain\"]\n", @@ -775,9 +1071,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['host1.cmh', 'host2.cmh', 'spine00.cmh', 'spine01.cmh', 'leaf00.cmh', 'leaf01.cmh'])" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "nr.filter(site=\"cmh\").inventory.hosts.keys()" ] @@ -791,9 +1098,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['spine00.cmh', 'spine01.cmh'])" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "nr.filter(site=\"cmh\", role=\"spine\").inventory.hosts.keys()" ] @@ -807,9 +1125,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['spine00.cmh', 'spine01.cmh'])" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "nr.filter(site=\"cmh\").filter(role=\"spine\").inventory.hosts.keys()" ] @@ -823,9 +1152,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['spine00.cmh', 'spine01.cmh'])" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "cmh = nr.filter(site=\"cmh\")\n", "cmh.filter(role=\"spine\").inventory.hosts.keys()" @@ -833,9 +1173,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['leaf00.cmh', 'leaf01.cmh'])" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "cmh.filter(role=\"leaf\").inventory.hosts.keys()" ] @@ -865,9 +1216,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['spine00.cmh', 'spine01.cmh', 'spine00.bma', 'spine01.bma'])" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "def has_long_name(host):\n", " return len(host.name) == 11\n", @@ -877,9 +1239,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['host1.cmh', 'host2.cmh', 'host1.bma', 'host2.bma'])" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Or a lambda function\n", "nr.filter(filter_func=lambda h: len(h.name) == 9).inventory.hosts.keys()" @@ -896,7 +1269,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -906,9 +1279,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['host1.cmh', 'host2.cmh', 'spine00.cmh', 'spine01.cmh', 'leaf00.cmh', 'leaf01.cmh'])\n" + ] + } + ], "source": [ "# hosts in group cmh\n", "cmh = nr.filter(F(groups__contains=\"cmh\"))\n", @@ -917,9 +1298,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['host1.cmh', 'host2.cmh', 'spine00.cmh', 'leaf00.cmh', 'host1.bma', 'host2.bma', 'spine00.bma', 'leaf00.bma'])\n" + ] + } + ], "source": [ "# devices running either linux or eos\n", "linux_or_eos = nr.filter(F(platform=\"linux\") | F(platform=\"eos\"))\n", @@ -928,9 +1317,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['spine00.cmh', 'spine01.cmh'])\n" + ] + } + ], "source": [ "# spines in cmh\n", "cmh_and_spine = nr.filter(F(groups__contains=\"cmh\") & F(role=\"spine\"))\n", @@ -939,9 +1336,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['host1.cmh', 'host2.cmh', 'leaf00.cmh', 'leaf01.cmh'])\n" + ] + } + ], "source": [ "# cmh devices that are not spines\n", "cmh_and_not_spine = nr.filter(F(groups__contains=\"cmh\") & ~F(role=\"spine\"))\n", @@ -957,9 +1362,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['host1.cmh'])\n" + ] + } + ], "source": [ "nested_string_asd = nr.filter(F(nested_data__a_string__contains=\"asd\"))\n", "print(nested_string_asd.inventory.hosts.keys())" @@ -967,9 +1380,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['host2.cmh'])\n" + ] + } + ], "source": [ "a_dict_element_equals = nr.filter(F(nested_data__a_dict__c=3))\n", "print(a_dict_element_equals.inventory.hosts.keys())" @@ -977,9 +1398,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['host1.cmh', 'host2.cmh'])\n" + ] + } + ], "source": [ "a_list_contains = nr.filter(F(nested_data__a_list__contains=2))\n", "print(a_list_contains.inventory.hosts.keys())" diff --git a/docs/howto/transforming_inventory_data.ipynb b/docs/howto/transforming_inventory_data.ipynb index 0781ea35..92cf6fb3 100644 --- a/docs/howto/transforming_inventory_data.ipynb +++ b/docs/howto/transforming_inventory_data.ipynb @@ -67,37 +67,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'defaults': {'connection_options': {},\n", - " 'data': {},\n", - " 'hostname': None,\n", - " 'password': None,\n", - " 'platform': None,\n", - " 'port': None,\n", - " 'username': None},\n", - " 'groups': {'defaults': {'connection_options': {},\n", - " 'data': {'a_default_attribute': 'my_default'},\n", - " 'groups': [],\n", - " 'hostname': None,\n", - " 'password': None,\n", - " 'platform': None,\n", - " 'port': None,\n", - " 'username': None}},\n", - " 'hosts': {'rtr00': {'connection_options': {},\n", - " 'data': {'user': 'automation_user'},\n", - " 'groups': [],\n", - " 'hostname': None,\n", - " 'password': None,\n", - " 'platform': None,\n", - " 'port': None,\n", - " 'username': 'automation_user'},\n", - " 'rtr01': {'connection_options': {},\n", - " 'data': {'user': 'automation_user'},\n", - " 'groups': [],\n", - " 'hostname': None,\n", - " 'password': None,\n", - " 'platform': None,\n", - " 'port': None,\n", - " 'username': 'automation_user'}}}\n" + "rtr00.username: automation_user\n", + "rtr01.username: automation_user\n" ] } ], @@ -119,7 +90,8 @@ " \"transform_function\": adapt_host_data,\n", " },\n", ")\n", - "pprint.pprint(nr.inventory.dict())" + "for name, host in nr.inventory.hosts.items():\n", + " print(f\"{name}.username: {host.username}\")" ] }, { @@ -224,37 +196,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'defaults': {'connection_options': {},\n", - " 'data': {},\n", - " 'hostname': None,\n", - " 'password': None,\n", - " 'platform': None,\n", - " 'port': None,\n", - " 'username': None},\n", - " 'groups': {'defaults': {'connection_options': {},\n", - " 'data': {'a_default_attribute': 'my_default'},\n", - " 'groups': [],\n", - " 'hostname': None,\n", - " 'password': None,\n", - " 'platform': None,\n", - " 'port': None,\n", - " 'username': None}},\n", - " 'hosts': {'rtr00': {'connection_options': {},\n", - " 'data': {'user': 'automation_user'},\n", - " 'groups': [],\n", - " 'hostname': None,\n", - " 'password': None,\n", - " 'platform': None,\n", - " 'port': None,\n", - " 'username': 'automation_user'},\n", - " 'rtr01': {'connection_options': {},\n", - " 'data': {'user': 'automation_user'},\n", - " 'groups': [],\n", - " 'hostname': None,\n", - " 'password': None,\n", - " 'platform': None,\n", - " 'port': None,\n", - " 'username': 'automation_user'}}}\n" + "rtr00.username: automation_user\n", + "rtr01.username: automation_user\n" ] } ], @@ -265,7 +208,8 @@ "nr = InitNornir(\n", " config_file=\"transforming_inventory_data/config.yaml\",\n", ")\n", - "pprint.pprint(nr.inventory.dict())" + "for name, host in nr.inventory.hosts.items():\n", + " print(f\"{name}.username: {host.username}\")" ] }, { diff --git a/docs/howto/writing_a_custom_inventory_plugin.ipynb b/docs/howto/writing_a_custom_inventory_plugin.ipynb index e3d4f206..d62ecdb7 100644 --- a/docs/howto/writing_a_custom_inventory_plugin.ipynb +++ b/docs/howto/writing_a_custom_inventory_plugin.ipynb @@ -18,7 +18,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "from nornir.core.inventory import Inventory\r\n", + "from nornir.core.deserializer.inventory import Inventory\r\n", "\r\n", "\r\n", "class MyInventory(Inventory):\r\n", @@ -48,7 +48,7 @@ " # passing the data to the parent class so the data is\r\n", " # transformed into actual Host/Group objects\r\n", " # and set default data for all hosts\r\n", - " super().__init__(hosts, groups, defaults, **kwargs)\r\n" + " super().__init__(hosts=hosts, groups=groups, defaults=defaults, **kwargs)\r\n" ] } ], @@ -114,6 +114,7 @@ ], "source": [ "from nornir.core import InitNornir\n", + "from nornir.core.deserializer.inventory import Inventory\n", "import pprint\n", "\n", "nr = InitNornir(\n", @@ -121,15 +122,8 @@ " \"plugin\": \"writing_a_custom_inventory_plugin.my_inventory.MyInventory\"\n", " }\n", ")\n", - "pprint.pprint(nr.inventory.dict())" + "pprint.pprint(Inventory.serialize(nr.inventory).dict())" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/docs/howto/writing_a_custom_inventory_plugin/my_inventory.py b/docs/howto/writing_a_custom_inventory_plugin/my_inventory.py index 5f747d04..3cdff8e6 100644 --- a/docs/howto/writing_a_custom_inventory_plugin/my_inventory.py +++ b/docs/howto/writing_a_custom_inventory_plugin/my_inventory.py @@ -1,4 +1,4 @@ -from nornir.core.inventory import Inventory +from nornir.core.deserializer.inventory import Inventory class MyInventory(Inventory): @@ -28,4 +28,4 @@ def __init__(self, **kwargs): # passing the data to the parent class so the data is # transformed into actual Host/Group objects # and set default data for all hosts - super().__init__(hosts, groups, defaults, **kwargs) + super().__init__(hosts=hosts, groups=groups, defaults=defaults, **kwargs) diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 204ff75a..cb295d42 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -189,11 +189,14 @@ def __getattribute__(self, name): else: return v + def __bool__(self): + return bool(self.name) + def __setitem__(self, item, value): self.data[item] = value def __len__(self): - return len(self.keys()) + return len(self._resolve_data().keys()) def __iter__(self): return self.data.__iter__() diff --git a/setup.cfg b/setup.cfg index 9be28f7d..ea33b94a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,7 +18,7 @@ skip = .tox/* max_line_length = 100 [tool:pytest] -#addopts = --cov=nornir --cov-report=term-missing -vs +addopts = --cov=nornir --cov-report=term-missing -vs python_paths = ./ [mypy] From 88115dab6834690d37dce5a757ac097279c86744 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 9 Sep 2018 12:16:39 +0200 Subject: [PATCH 056/109] progress --- nornir/core/deserializer/inventory.py | 70 +++++++++++++++++++++++---- nornir/core/inventory.py | 70 ++++++++++++++------------- nornir/plugins/inventory/ansible.py | 4 ++ 3 files changed, 100 insertions(+), 44 deletions(-) diff --git a/nornir/core/deserializer/inventory.py b/nornir/core/deserializer/inventory.py index 3346b9dc..2d419106 100644 --- a/nornir/core/deserializer/inventory.py +++ b/nornir/core/deserializer/inventory.py @@ -4,10 +4,6 @@ from pydantic import BaseModel -GroupsDict = None # DELETEME -HostsDict = None # DELETEME -VarsDict = None # DELETEME - class BaseAttributes(BaseModel): hostname: Optional[str] = None @@ -29,12 +25,57 @@ class InventoryElement(BaseAttributes): data: Dict[str, Any] = {} connection_options: Dict[str, ConnectionOptions] = {} + @classmethod + def deserialize( + cls, + name: str, + hostname: Optional[str] = None, + port: Optional[int] = None, + username: Optional[str] = None, + password: Optional[str] = None, + platform: Optional[str] = None, + groups: Optional[List[str]] = None, + data: Optional[Dict[str, Any]] = None, + connection_options: Optional[Dict[str, ConnectionOptions]] = None, + defaults: inventory.Defaults = None, + ) -> Dict[str, Any]: + parent_groups = inventory.ParentGroups(groups) + connection_options = connection_options or {} + conn_opts = { + k: inventory.ConnectionOptions(**v) for k, v in connection_options.items() + } + return { + "name": name, + "hostname": hostname, + "port": port, + "username": username, + "password": password, + "platform": platform, + "groups": parent_groups, + "data": data, + "connection_options": conn_opts, + "defaults": defaults, + } + + @classmethod + def deserialize_host(cls, **kwargs) -> inventory.Host: + return inventory.Host(**cls.deserialize(**kwargs)) + + @classmethod + def deserialize_group(cls, **kwargs) -> inventory.Group: + return inventory.Group(**cls.deserialize(**kwargs)) + @classmethod def serialize(cls, e: Union[inventory.Host, inventory.Group]) -> "InventoryElement": d = {} for f in cls.__fields__: d[f] = object.__getattribute__(e, f) d["groups"] = list(d["groups"]) + + d["connection_options"] = { + k: {f: getattr(v, f) for f in v.__recursive_slots__()} + for k, v in d["connection_options"].items() + } return InventoryElement(**d) @@ -47,29 +88,38 @@ def serialize(cls, defaults: inventory.Defaults) -> "InventoryElement": d = {} for f in cls.__fields__: d[f] = getattr(defaults, f) + + d["connection_options"] = { + k: {f: getattr(v, f) for f in v.__recursive_slots__()} + for k, v in d["connection_options"].items() + } return Defaults(**d) class Inventory(BaseModel): hosts: Dict[str, InventoryElement] - groups: Dict[str, InventoryElement] = {} - defaults: Defaults = {} + groups: Dict[str, InventoryElement] + defaults: Defaults @classmethod def deserialize(cls, transform_function=None, *args, **kwargs): deserialized = cls(*args, **kwargs) defaults = inventory.Defaults(**deserialized.defaults.dict()) + for k, v in defaults.connection_options.items(): + defaults.connection_options[k] = inventory.ConnectionOptions(**v) hosts = inventory.Hosts() for n, h in deserialized.hosts.items(): - h.groups = inventory.ParentGroups(h.groups) - hosts[n] = inventory.Host(defaults=defaults, name=n, **h.dict()) + hosts[n] = InventoryElement.deserialize_host( + defaults=defaults, name=n, **h.dict() + ) groups = inventory.Groups() for n, g in deserialized.groups.items(): - g.groups = inventory.ParentGroups(g.groups) - groups[n] = inventory.Group(defaults=defaults, name=n, **g.dict()) + groups[n] = InventoryElement.deserialize_group( + defaults=defaults, name=n, **g.dict() + ) return inventory.Inventory( hosts=hosts, diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index cb295d42..ac48eaa1 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -20,7 +20,7 @@ def __init__( username: Optional[str] = None, password: Optional[str] = None, platform: Optional[str] = None, - ): + ) -> None: self.hostname = hostname self.port = port self.username = username @@ -43,7 +43,7 @@ def dict(self): class ConnectionOptions(BaseAttributes): __slots__ = ("extras",) - def __init__(self, extras: Optional[Dict[str, Any]] = None, **kwargs): + def __init__(self, extras: Optional[Dict[str, Any]] = None, **kwargs) -> None: self.extras = extras or {} super().__init__(**kwargs) @@ -51,7 +51,8 @@ def __init__(self, extras: Optional[Dict[str, Any]] = None, **kwargs): class ParentGroups(UserList): __slots__ = "refs" - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: + self.data: List[str] = [] super().__init__(*args, **kwargs) self.refs: List["Group"] = kwargs.get("refs", []) @@ -72,7 +73,7 @@ def __init__( connection_options: Optional[Dict[str, ConnectionOptions]] = None, config: Optional[Config] = None, **kwargs, - ): + ) -> None: self.groups = groups or ParentGroups() self.data = data or {} self.connection_options = connection_options or {} @@ -88,7 +89,7 @@ def __init__( data: Optional[Dict[str, Any]] = None, connection_options: Optional[Dict[str, ConnectionOptions]] = None, **kwargs, - ): + ) -> None: self.data = data or {} self.connection_options = connection_options or {} super().__init__(**kwargs) @@ -100,10 +101,10 @@ class Host(InventoryElement): def __init__( self, name: str, - defaults: Optional[InventoryElement] = None, + defaults: Optional[Defaults] = None, config: Optional[Config] = None, **kwargs, - ): + ) -> None: self.name = name self.defaults = defaults or Defaults() self.connections: Connections = Connections() @@ -225,32 +226,34 @@ def get(self, item, default=None): def get_connection_parameters( self, connection: Optional[str] = None - ) -> Dict[str, Any]: + ) -> ConnectionOptions: if not connection: - d = { - "hostname": self.hostname, - "port": self.port, - "username": self.username, - "password": self.password, - "platform": self.platform, - "extras": {}, - } + d = ConnectionOptions( + hostname=self.hostname, + port=self.port, + username=self.username, + password=self.password, + platform=self.platform, + extras={}, + ) else: - d = self._get_connection_options_recursively(connection) - if d is not None: - return ConnectionOptions(**d) + r = self._get_connection_options_recursively(connection) + if r is not None: + return r else: - d = { - "hostname": self.hostname, - "port": self.port, - "username": self.username, - "password": self.password, - "platform": self.platform, - "extras": {}, - } - return ConnectionOptions(**d) - - def _get_connection_options_recursively(self, connection: str) -> Dict[str, Any]: + d = ConnectionOptions( + hostname=self.hostname, + port=self.port, + username=self.username, + password=self.password, + platform=self.platform, + extras={}, + ) + return d + + def _get_connection_options_recursively( + self, connection: str + ) -> Optional[ConnectionOptions]: p = self.connection_options.get(connection) if p is None: for g in self.groups.refs: @@ -385,17 +388,16 @@ def __init__( defaults: Optional[Defaults] = None, config: Optional[Config] = None, transform_function=None, - ): + ) -> None: self.hosts = hosts - self.groups = groups + self.groups = groups or Groups() + self.defaults = defaults or Defaults() for host in self.hosts.values(): host.groups.refs = [self.groups[p] for p in host.groups] for group in self.groups.values(): group.groups.refs = [self.groups[p] for p in group.groups] - self.defaults = defaults - if transform_function: for h in self.hosts.values(): transform_function(h) diff --git a/nornir/plugins/inventory/ansible.py b/nornir/plugins/inventory/ansible.py index 2166997c..c00079a1 100644 --- a/nornir/plugins/inventory/ansible.py +++ b/nornir/plugins/inventory/ansible.py @@ -6,7 +6,10 @@ from typing import Dict, Any, Tuple, Optional, cast, Union, MutableMapping, DefaultDict import ruamel.yaml + + from mypy_extensions import TypedDict + from ruamel.yaml.scanner import ScannerError from ruamel.yaml.composer import ComposerError @@ -14,6 +17,7 @@ VARS_FILENAME_EXTENSIONS = ["", ".yml", ".yaml"] + YAML = ruamel.yaml.YAML(typ="safe") logger = logging.getLogger("nornir") From 4a3efbed35d6bf94c112cadae4aa57bcedfcf45c Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 9 Sep 2018 13:13:56 +0200 Subject: [PATCH 057/109] fix mypy --- nornir/core/deserializer/inventory.py | 12 +++++++++--- nornir/core/inventory.py | 4 ---- nornir/plugins/inventory/ansible.py | 7 ++++++- nornir/plugins/inventory/netbox.py | 5 +++-- nornir/plugins/inventory/simple.py | 6 +++--- setup.cfg | 9 +++++++++ 6 files changed, 30 insertions(+), 13 deletions(-) diff --git a/nornir/core/deserializer/inventory.py b/nornir/core/deserializer/inventory.py index 2d419106..a3a93c7b 100644 --- a/nornir/core/deserializer/inventory.py +++ b/nornir/core/deserializer/inventory.py @@ -5,6 +5,11 @@ from pydantic import BaseModel +VarsDict = Dict[str, Any] +HostsDict = Dict[str, VarsDict] +GroupsDict = Dict[str, VarsDict] + + class BaseAttributes(BaseModel): hostname: Optional[str] = None port: Optional[int] = None @@ -105,9 +110,10 @@ class Inventory(BaseModel): def deserialize(cls, transform_function=None, *args, **kwargs): deserialized = cls(*args, **kwargs) - defaults = inventory.Defaults(**deserialized.defaults.dict()) - for k, v in defaults.connection_options.items(): - defaults.connection_options[k] = inventory.ConnectionOptions(**v) + defaults_dict = deserialized.defaults.dict() + for k, v in defaults_dict["connection_options"].items(): + defaults_dict["connection_options"][k] = inventory.ConnectionOptions(**v) + defaults = inventory.Defaults(**defaults_dict) hosts = inventory.Hosts() for n, h in deserialized.hosts.items(): diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index ac48eaa1..b3a47097 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -5,10 +5,6 @@ from nornir.core.connections import Connections from nornir.core.exceptions import ConnectionAlreadyOpen, ConnectionNotOpen -GroupsDict = None # DELETEME -HostsDict = None # DELETEME -VarsDict = None # DELETEME - class BaseAttributes(object): __slots__ = ("hostname", "port", "username", "password", "platform") diff --git a/nornir/plugins/inventory/ansible.py b/nornir/plugins/inventory/ansible.py index c00079a1..a5d45226 100644 --- a/nornir/plugins/inventory/ansible.py +++ b/nornir/plugins/inventory/ansible.py @@ -13,7 +13,12 @@ from ruamel.yaml.scanner import ScannerError from ruamel.yaml.composer import ComposerError -from nornir.core.inventory import Inventory, VarsDict, GroupsDict, HostsDict +from nornir.core.deserializer.inventory import ( + Inventory, + VarsDict, + GroupsDict, + HostsDict, +) VARS_FILENAME_EXTENSIONS = ["", ".yml", ".yaml"] diff --git a/nornir/plugins/inventory/netbox.py b/nornir/plugins/inventory/netbox.py index 628098c6..92488367 100644 --- a/nornir/plugins/inventory/netbox.py +++ b/nornir/plugins/inventory/netbox.py @@ -1,6 +1,7 @@ import os -from nornir.core.deserializer.inventory import Inventory +from nornir.core.deserializer.inventory import Inventory, HostsDict + import requests @@ -28,7 +29,7 @@ def __init__( hosts = {} for d in nb_devices["results"]: - host = {"data": {}} + host: HostsDict = {"data": {}} # Add value for IP address if d.get("primary_ip", {}): diff --git a/nornir/plugins/inventory/simple.py b/nornir/plugins/inventory/simple.py index b67fd1de..25d5bb35 100644 --- a/nornir/plugins/inventory/simple.py +++ b/nornir/plugins/inventory/simple.py @@ -1,7 +1,7 @@ import logging import os -from nornir.core.deserializer.inventory import Inventory +from nornir.core.deserializer.inventory import Inventory, VarsDict, GroupsDict import ruamel.yaml @@ -19,7 +19,7 @@ def __init__( with open(host_file, "r") as f: hosts = yml.load(f) - groups = {} + groups: GroupsDict = {} if group_file: if os.path.exists(group_file): with open(group_file, "r") as f: @@ -28,7 +28,7 @@ def __init__( logging.warning("{}: doesn't exist".format(group_file)) groups = {} - defaults = {} + defaults: VarsDict = {} if defaults_file: if os.path.exists(defaults_file): with open(defaults_file, "r") as f: diff --git a/setup.cfg b/setup.cfg index ea33b94a..26418df8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,6 +46,15 @@ strict_optional = True warn_unused_ignores = True ignore_errors = False +[mypy-nornir.core.deserializer.*] +check_untyped_defs = True +disallow_any_generics = True +# Turn on the next flag once the whole codebase is annotated (Phase 2) +# disallow_untyped_calls = True +strict_optional = True +warn_unused_ignores = True +ignore_errors = False + [mypy-nornir.plugins.*] check_untyped_defs = True disallow_any_generics = True From dfea3752cff217c3d604f2eaf351e649b7d53e43 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 9 Sep 2018 13:32:56 +0200 Subject: [PATCH 058/109] we need to fix doc references --- docs/conf.py | 19 ------ docs/configuration/generated/.placeholder | 0 docs/configuration/index.rst | 8 +-- docs/howto/handling_connections.ipynb | 74 ++++++++++++++++------- setup.cfg | 6 ++ tox.ini | 1 + 6 files changed, 59 insertions(+), 49 deletions(-) delete mode 100644 docs/configuration/generated/.placeholder diff --git a/docs/conf.py b/docs/conf.py index 25cab2c7..42239c89 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -26,8 +26,6 @@ sys.path.insert(0, os.path.abspath("../")) -from nornir.core.configuration import CONF # noqa - # -- General configuration ------------------------------------------------ BASEPATH = os.path.dirname(__file__) @@ -173,19 +171,6 @@ ] -def build_configuration_parameters(app): - """Create documentation for configuration parameters.""" - - env = Environment(loader=FileSystemLoader("{0}/_data_templates".format(BASEPATH))) - template_file = env.get_template("configuration-parameters.j2") - data = {} - data["params"] = CONF - rendered_template = template_file.render(**data) - output_dir = "{0}/configuration/generated".format(BASEPATH) - with open("{}/parameters.rst".format(output_dir), "w") as f: - f.write(rendered_template) - - def skip_slots(app, what, name, obj, skip, options): if obj.__class__.__name__ == "member_descriptor": return True @@ -194,9 +179,5 @@ def skip_slots(app, what, name, obj, skip, options): def setup(app): """Map methods to states of the documentation build.""" - app.connect("builder-inited", build_configuration_parameters) app.connect("autodoc-skip-member", skip_slots) app.add_stylesheet("css/custom.css") - - -build_configuration_parameters(None) diff --git a/docs/configuration/generated/.placeholder b/docs/configuration/generated/.placeholder deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/configuration/index.rst b/docs/configuration/index.rst index 7f5f84ec..6d292c38 100644 --- a/docs/configuration/index.rst +++ b/docs/configuration/index.rst @@ -1,10 +1,4 @@ Configuration ============= -Each configuration parameter are applied in the following order: - -1. Environment variable -2. Parameter in configuration file / object -3. Default value - -.. include:: generated/parameters.rst +# TODO diff --git a/docs/howto/handling_connections.ipynb b/docs/howto/handling_connections.ipynb index d8912a2f..486f506c 100644 --- a/docs/howto/handling_connections.ipynb +++ b/docs/howto/handling_connections.ipynb @@ -124,29 +124,57 @@ "source": [ "## Specifying connection parameters\n", "\n", - "When using the [open_connection](../ref/api/inventory.rst#nornir.core.inventory.Host.open_connection) you can specify any parameters you want. If you don't, or if you let nornir open the connection automatically, nornir will read those parameters from the inventory. You can specify standard attributes at the object level if you want to reuse them across different connections or you can override them for each connection:\n", - "\n", - "```\n", - "my_host:\n", - " # standard parameters that will be reused across different connections\n", - " username: my_user\n", - " password: my_password\n", - " port: 22\n", - " platform: linux\n", - " napalm_options:\n", - " # standard parameters that will be used only for the napalm connection\n", - " # missing parameters will be read from the top level\n", - " port: 443\n", - " platform: eos\n", - " connection_options:\n", - " # advanced options that are specific to this connection type\n", - " optional_args:\n", - " eos_autoComplete: true\n", - " netmiko_options:\n", - " platform: arista_eos\n", - " connection_options:\n", - " global_delay: 2\n", - "```" + "When using the [open_connection](../ref/api/inventory.rst#nornir.core.inventory.Host.open_connection) you can specify any parameters you want. If you don't, or if you let nornir open the connection automatically, nornir will read those parameters from the inventory. You can specify standard attributes at the object level if you want to reuse them across different connections or you can override them for each connection. For example:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dev1.group_1:\r\n", + " port: 65001\r\n", + " hostname:\r\n", + " username:\r\n", + " password: a_password\r\n", + " platform: eos\r\n", + " data:\r\n", + " my_var: comes_from_dev1.group_1\r\n", + " www_server: nginx\r\n", + " role: www\r\n", + " nested_data:\r\n", + " a_dict:\r\n", + " a: 1\r\n", + " b: 2\r\n", + " a_list: [1, 2]\r\n", + " a_string: asdasd\r\n", + " groups:\r\n", + " - group_1\r\n", + " connection_options:\r\n", + " paramiko:\r\n", + " port: 65001\r\n", + " hostname: 127.0.0.1\r\n", + " username: root\r\n", + " password: docker\r\n", + " platform: linux\r\n", + " extras: {}\r\n", + " dummy:\r\n", + " hostname: dummy_from_host\r\n", + " port:\r\n", + " username:\r\n", + " password:\r\n", + " platform:\r\n", + " extras:\r\n", + " blah: from_host\r\n" + ] + } + ], + "source": [ + "!sed '2,35!d' ../../tests/inventory_data/hosts.yaml" ] } ], diff --git a/setup.cfg b/setup.cfg index 26418df8..6e4d920b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,6 +17,12 @@ skip = .tox/* [pylama:pep8] max_line_length = 100 +[pycodestyle] +ignore = D203,C901 +exclude = .git,__pycache__,build,dist +max-complexity = 10 +max-line-length = 100 + [tool:pytest] addopts = --cov=nornir --cov-report=term-missing -vs python_paths = ./ diff --git a/tox.ini b/tox.ini index 3b0bd624..4a7ea34e 100644 --- a/tox.ini +++ b/tox.ini @@ -24,6 +24,7 @@ deps = basepython = python3.6 commands = sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html + # TODO REPLACE with: sphinx-build -n -E -q -N -b dummy -d docs/_build/doctrees docs asd [testenv:pylama] deps = From 7127dbb5c8ef5ac93fe8392ef88d561ee714bec4 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 9 Sep 2018 13:50:50 +0200 Subject: [PATCH 059/109] fix circular reference --- docs/howto/inventory.ipynb | 8 ++++---- docs/ref/api/inventory.rst | 7 +++++++ docs/ref/api/nornir.rst | 2 +- docs/tutorials/intro/failed_tasks.ipynb | 4 ++-- nornir/core/__init__.py | 3 --- nornir/plugins/connections/__init__.py | 3 +++ 6 files changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/howto/inventory.ipynb b/docs/howto/inventory.ipynb index aec08a41..4558cec6 100644 --- a/docs/howto/inventory.ipynb +++ b/docs/howto/inventory.ipynb @@ -18,9 +18,9 @@ "source": [ "## Inventory\n", "\n", - "The Inventory is arguably the most important piece of nornir. Let's see how it works. To begin with the [inventory](../../ref/api/inventory.rst#nornir.core.inventory.Inventory) is comprised of [hosts](../../ref/api/inventory.rst#nornir.core.inventory.Host), [groups](../../ref/api/inventory.rst#nornir.core.inventory.Group) and [defaults](../../ref/api/inventory.rst#nornir.core.inventory.Defaults).\n", + "The Inventory is arguably the most important piece of nornir. Let's see how it works. To begin with the [inventory](../ref/api/inventory.rst#nornir.core.inventory.Inventory) is comprised of [hosts](../ref/api/inventory.rst#nornir.core.inventory.Host), [groups](../ref/api/inventory.rst#nornir.core.inventory.Group) and [defaults](../ref/api/inventory.rst#nornir.core.inventory.Defaults).\n", "\n", - "In this tutorial we are using the [SimpleInventory](../../plugins/inventory/simple.rst#nornir.plugins.inventory.simple.SimpleInventory) plugin. This inventory plugin stores all the relevant data in three files. Let's start by checking them:" + "In this tutorial we are using the [SimpleInventory](../plugins/inventory/simple.rst#nornir.plugins.inventory.simple.SimpleInventory) plugin. This inventory plugin stores all the relevant data in three files. Let's start by checking them:" ] }, { @@ -645,7 +645,7 @@ "\n", "### Accessing the inventory\n", "\n", - "You can access the [inventory](../../ref/api/inventory.rst#nornir.core.inventory.Inventory) with the `inventory` attribute:" + "You can access the [inventory](../ref/api/inventory.rst#nornir.core.inventory.Inventory) with the `inventory` attribute:" ] }, { @@ -1211,7 +1211,7 @@ "\n", "##### Filter functions\n", "\n", - "The ``filter_func`` parameter let's you run your own code to filter the hosts. The function signature is as simple as ``my_func(host)`` where host is an object of type [Host](../../ref/api/inventory.rst#nornir.core.inventory.Host) and it has to return either ``True`` or ``False`` to indicate if you want to host or not." + "The ``filter_func`` parameter let's you run your own code to filter the hosts. The function signature is as simple as ``my_func(host)`` where host is an object of type [Host](../ref/api/inventory.rst#nornir.core.inventory.Host) and it has to return either ``True`` or ``False`` to indicate if you want to host or not." ] }, { diff --git a/docs/ref/api/inventory.rst b/docs/ref/api/inventory.rst index df93a6a9..a55646ce 100644 --- a/docs/ref/api/inventory.rst +++ b/docs/ref/api/inventory.rst @@ -19,3 +19,10 @@ Group ===== .. autoclass:: nornir.core.inventory.Group + +Defaults +======== + +.. autoclass:: nornir.core.inventory.Defaults + :members: + :undoc-members: diff --git a/docs/ref/api/nornir.rst b/docs/ref/api/nornir.rst index c1b97d30..197f28df 100644 --- a/docs/ref/api/nornir.rst +++ b/docs/ref/api/nornir.rst @@ -16,6 +16,6 @@ Nornir Data ---- -.. autoclass:: nornir.core.Data +.. autoclass:: nornir.core.state.GlobalState :members: :undoc-members: diff --git a/docs/tutorials/intro/failed_tasks.ipynb b/docs/tutorials/intro/failed_tasks.ipynb index 26a60b14..762e2213 100644 --- a/docs/tutorials/intro/failed_tasks.ipynb +++ b/docs/tutorials/intro/failed_tasks.ipynb @@ -409,7 +409,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "To achieve this `nornir` keeps a list failed hosts in it's shared [data](../../ref/api/nornir.rst#nornir.core.Data) object:" + "To achieve this `nornir` keeps a list failed hosts in it's shared [data](../../ref/api/nornir.rst#nornir.core.state.GlobalState) object:" ] }, { @@ -436,7 +436,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "If you want to mark some hosts as succeeded and make them back eligible for future tasks you can do it individually per host with the function [recover_host](../../ref/api/nornir.rst#nornir.core.Data.recover_host) or reset the list completely with [reset_failed_hosts](../../ref/api/nornir.rst#nornir.core.Data.reset_failed_hosts):" + "If you want to mark some hosts as succeeded and make them back eligible for future tasks you can do it individually per host with the function [recover_host](../../ref/api/nornir.rst#nornir.core.state.GlobalState.recover_host) or reset the list completely with [reset_failed_hosts](../../ref/api/nornir.rst#nornir.core.state.GlobalState.reset_failed_hosts):" ] }, { diff --git a/nornir/core/__init__.py b/nornir/core/__init__.py index 3be4e127..cacd8da9 100644 --- a/nornir/core/__init__.py +++ b/nornir/core/__init__.py @@ -5,9 +5,6 @@ from nornir.core.configuration import Config from nornir.core.state import GlobalState from nornir.core.task import AggregatedResult, Task -from nornir.plugins.connections import register_default_connection_plugins - -register_default_connection_plugins() class Nornir(object): diff --git a/nornir/plugins/connections/__init__.py b/nornir/plugins/connections/__init__.py index fb149889..220748f1 100644 --- a/nornir/plugins/connections/__init__.py +++ b/nornir/plugins/connections/__init__.py @@ -8,3 +8,6 @@ def register_default_connection_plugins() -> None: Connections.register("napalm", Napalm) Connections.register("netmiko", Netmiko) Connections.register("paramiko", Paramiko) + + +register_default_connection_plugins() From 7d8aa5e6f3711d1e1807666875609d51cca07d5d Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 9 Sep 2018 14:39:54 +0200 Subject: [PATCH 060/109] fix cyclic import --- docs/conf.py | 3 -- docs/howto/advanced_filtering.ipynb | 2 +- docs/howto/handling_connections.ipynb | 7 ++-- docs/howto/inventory.ipynb | 2 +- docs/howto/transforming_inventory_data.ipynb | 6 +-- .../writing_a_custom_inventory_plugin.ipynb | 2 +- docs/index.rst | 2 +- docs/ref/api/nornir.rst | 3 +- docs/tutorials/intro/executing_tasks.ipynb | 2 +- docs/tutorials/intro/failed_tasks.ipynb | 2 +- docs/tutorials/intro/grouping_tasks.ipynb | 2 +- .../tutorials/intro/initializing_nornir.ipynb | 6 +-- docs/tutorials/intro/inventory.ipynb | 2 +- docs/tutorials/intro/task_results.ipynb | 2 +- docs/upgrading/1_to_2.rst | 11 +++++ nornir/__init__.py | 5 +++ nornir/core/__init__.py | 26 ------------ nornir/init_nornir.py | 42 +++++++++++++++++++ nornir/plugins/connections/__init__.py | 13 ------ tests/conftest.py | 2 +- tests/core/test_InitNornir.py | 2 +- tests/core/test_connections.py | 2 +- .../plugins/tasks/networking/test_tcp_ping.py | 2 +- tox.ini | 2 +- 24 files changed, 84 insertions(+), 66 deletions(-) create mode 100644 nornir/init_nornir.py diff --git a/docs/conf.py b/docs/conf.py index 42239c89..eab549db 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,9 +20,6 @@ import os import sys -from jinja2 import Environment -from jinja2 import FileSystemLoader - sys.path.insert(0, os.path.abspath("../")) diff --git a/docs/howto/advanced_filtering.ipynb b/docs/howto/advanced_filtering.ipynb index 4e4651d9..9ee6e1a1 100644 --- a/docs/howto/advanced_filtering.ipynb +++ b/docs/howto/advanced_filtering.ipynb @@ -15,7 +15,7 @@ "metadata": {}, "outputs": [], "source": [ - "from nornir.core import InitNornir\n", + "from nornir import InitNornir\n", "from nornir.core.filter import F\n", "\n", "nr = InitNornir(config_file=\"advanced_filtering/config.yaml\")" diff --git a/docs/howto/handling_connections.ipynb b/docs/howto/handling_connections.ipynb index 486f506c..e8986ef9 100644 --- a/docs/howto/handling_connections.ipynb +++ b/docs/howto/handling_connections.ipynb @@ -17,7 +17,7 @@ "metadata": {}, "outputs": [], "source": [ - "from nornir.core import InitNornir\n", + "from nornir import InitNornir\n", "from nornir.plugins.functions.text import print_result\n", "from nornir.plugins.tasks.networking import napalm_get" ] @@ -129,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -169,7 +169,8 @@ " password:\r\n", " platform:\r\n", " extras:\r\n", - " blah: from_host\r\n" + " blah: from_host\r\n", + "\u001b[0m\u001b[0m" ] } ], diff --git a/docs/howto/inventory.ipynb b/docs/howto/inventory.ipynb index 4558cec6..39993fff 100644 --- a/docs/howto/inventory.ipynb +++ b/docs/howto/inventory.ipynb @@ -654,7 +654,7 @@ "metadata": {}, "outputs": [], "source": [ - "from nornir.core import InitNornir\n", + "from nornir import InitNornir\n", "nr = InitNornir(\n", " inventory={\n", " \"options\": {\n", diff --git a/docs/howto/transforming_inventory_data.ipynb b/docs/howto/transforming_inventory_data.ipynb index 92cf6fb3..172a5bc6 100644 --- a/docs/howto/transforming_inventory_data.ipynb +++ b/docs/howto/transforming_inventory_data.ipynb @@ -73,7 +73,7 @@ } ], "source": [ - "from nornir.core import InitNornir\n", + "from nornir import InitNornir\n", "import pprint\n", "\n", "def adapt_host_data(host):\n", @@ -202,7 +202,7 @@ } ], "source": [ - "from nornir.core import InitNornir\n", + "from nornir import InitNornir\n", "import pprint\n", "\n", "nr = InitNornir(\n", @@ -237,7 +237,7 @@ } ], "source": [ - "from nornir.core import InitNornir\n", + "from nornir import InitNornir\n", "\n", "# let's pretend we used raw_input or something like that\n", "# password = raw_input(\"Please, enter password: \")\n", diff --git a/docs/howto/writing_a_custom_inventory_plugin.ipynb b/docs/howto/writing_a_custom_inventory_plugin.ipynb index d62ecdb7..abd89c48 100644 --- a/docs/howto/writing_a_custom_inventory_plugin.ipynb +++ b/docs/howto/writing_a_custom_inventory_plugin.ipynb @@ -113,7 +113,7 @@ } ], "source": [ - "from nornir.core import InitNornir\n", + "from nornir import InitNornir\n", "from nornir.core.deserializer.inventory import Inventory\n", "import pprint\n", "\n", diff --git a/docs/index.rst b/docs/index.rst index 58ebd6d0..83486ef9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,7 +34,7 @@ A first glance Here is an example on how to quickly build a runbook leveraging Nornir to retrieve information from the network:: - from nornir.core import InitNornir + from nornir import InitNornir from nornir.plugins.functions.text import print_result from nornir.plugins.tasks.networking import napalm_get diff --git a/docs/ref/api/nornir.rst b/docs/ref/api/nornir.rst index 197f28df..e052a71f 100644 --- a/docs/ref/api/nornir.rst +++ b/docs/ref/api/nornir.rst @@ -4,7 +4,8 @@ Core InitNornir ---------- -.. automethod:: nornir.core.InitNornir +.. automethod:: nornir.InitNornir + :module: nornir.init_nornir Nornir ------ diff --git a/docs/tutorials/intro/executing_tasks.ipynb b/docs/tutorials/intro/executing_tasks.ipynb index b6e50133..5928880e 100644 --- a/docs/tutorials/intro/executing_tasks.ipynb +++ b/docs/tutorials/intro/executing_tasks.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "outputs": [], "source": [ - "from nornir.core import InitNornir\n", + "from nornir import InitNornir\n", "nr = InitNornir(config_file=\"config.yaml\")" ] }, diff --git a/docs/tutorials/intro/failed_tasks.ipynb b/docs/tutorials/intro/failed_tasks.ipynb index 762e2213..de15cbd7 100644 --- a/docs/tutorials/intro/failed_tasks.ipynb +++ b/docs/tutorials/intro/failed_tasks.ipynb @@ -17,7 +17,7 @@ "metadata": {}, "outputs": [], "source": [ - "from nornir.core import InitNornir\n", + "from nornir import InitNornir\n", "from nornir.plugins.tasks import networking, text\n", "from nornir.plugins.functions.text import print_result\n", "\n", diff --git a/docs/tutorials/intro/grouping_tasks.ipynb b/docs/tutorials/intro/grouping_tasks.ipynb index 973e49b7..3f20083a 100644 --- a/docs/tutorials/intro/grouping_tasks.ipynb +++ b/docs/tutorials/intro/grouping_tasks.ipynb @@ -21,7 +21,7 @@ "metadata": {}, "outputs": [], "source": [ - "from nornir.core import InitNornir\n", + "from nornir import InitNornir\n", "from nornir.plugins.tasks import networking, text\n", "from nornir.plugins.functions.text import print_title, print_result\n", "\n", diff --git a/docs/tutorials/intro/initializing_nornir.ipynb b/docs/tutorials/intro/initializing_nornir.ipynb index ae0d5c0c..f7ff577a 100644 --- a/docs/tutorials/intro/initializing_nornir.ipynb +++ b/docs/tutorials/intro/initializing_nornir.ipynb @@ -146,7 +146,7 @@ "metadata": {}, "outputs": [], "source": [ - "from nornir.core import InitNornir\n", + "from nornir import InitNornir\n", "nr = InitNornir(config_file=\"config.yaml\")" ] }, @@ -163,7 +163,7 @@ "metadata": {}, "outputs": [], "source": [ - "from nornir.core import InitNornir\n", + "from nornir import InitNornir\n", "nr = InitNornir(num_workers=100,\n", " inventory=\"nornir.plugins.inventory.simple.SimpleInventory\",\n", " SimpleInventory={\"host_file\": \"inventory/hosts.yaml\",\n", @@ -183,7 +183,7 @@ "metadata": {}, "outputs": [], "source": [ - "from nornir.core import InitNornir\n", + "from nornir import InitNornir\n", "nr = InitNornir(num_workers=50, config_file=\"config.yaml\")" ] }, diff --git a/docs/tutorials/intro/inventory.ipynb b/docs/tutorials/intro/inventory.ipynb index cc4fa550..15153ed8 100644 --- a/docs/tutorials/intro/inventory.ipynb +++ b/docs/tutorials/intro/inventory.ipynb @@ -429,7 +429,7 @@ } ], "source": [ - "from nornir.core import InitNornir\n", + "from nornir import InitNornir\n", "nr = InitNornir(config_file=\"config.yaml\")\n", "\n", "nr.inventory" diff --git a/docs/tutorials/intro/task_results.ipynb b/docs/tutorials/intro/task_results.ipynb index 048a20c6..6d4dcac9 100644 --- a/docs/tutorials/intro/task_results.ipynb +++ b/docs/tutorials/intro/task_results.ipynb @@ -17,7 +17,7 @@ "metadata": {}, "outputs": [], "source": [ - "from nornir.core import InitNornir\n", + "from nornir import InitNornir\n", "from nornir.plugins.tasks import networking, text\n", "from nornir.plugins.functions.text import print_title\n", "import logging\n", diff --git a/docs/upgrading/1_to_2.rst b/docs/upgrading/1_to_2.rst index c129672c..1ce7e92f 100644 --- a/docs/upgrading/1_to_2.rst +++ b/docs/upgrading/1_to_2.rst @@ -13,3 +13,14 @@ When specifying connection parameters, in nornir 1.x those parameters where spec * ``platform`` (which replaces both ``os`` and ``network_operating_system``) You can check the following how to for more details on `how to <../howto/handling_connections.rst>`_ use these parameters. + +Changed to path importing ``InitNornir`` +---------------------------------------- + +In order to import ``InitNornir`` correctly you have to change the old path:: + + from nornir.core import InitNornir + +to:: + + from nornir import InitNornir diff --git a/nornir/__init__.py b/nornir/__init__.py index 593894db..7306fbdc 100644 --- a/nornir/__init__.py +++ b/nornir/__init__.py @@ -1,6 +1,11 @@ +from nornir.init_nornir import InitNornir import pkg_resources + try: __version__ = pkg_resources.get_distribution("nornir").version except pkg_resources.DistributionNotFound: __version__ = "Not installed" + + +__all__ = ("InitNornir", "__version__") diff --git a/nornir/core/__init__.py b/nornir/core/__init__.py index cacd8da9..68671c15 100644 --- a/nornir/core/__init__.py +++ b/nornir/core/__init__.py @@ -170,29 +170,3 @@ def validate(cls, v): @property def state(self): return GlobalState - - -def InitNornir(config_file="", dry_run=False, configure_logging=True, **kwargs): - """ - Arguments: - config_file(str): Path to the configuration file (optional) - dry_run(bool): Whether to simulate changes or not - **kwargs: Extra information to pass to the - :obj:`nornir.core.configuration.Config` object - - Returns: - :obj:`nornir.core.Nornir`: fully instantiated and configured - """ - conf = Config(path=config_file, **kwargs) - GlobalState.dry_run = dry_run - - if configure_logging: - conf.logging.configure() - - inv_class = conf.inventory.get_plugin() - transform_function = conf.inventory.get_transform_function() - inv = inv_class.deserialize( - transform_function=transform_function, config=conf, **conf.inventory.options - ) - - return Nornir(inventory=inv, _config=conf) diff --git a/nornir/init_nornir.py b/nornir/init_nornir.py new file mode 100644 index 00000000..008b4e1f --- /dev/null +++ b/nornir/init_nornir.py @@ -0,0 +1,42 @@ +from nornir.core import Nornir +from nornir.core.configuration import Config +from nornir.core.state import GlobalState +from nornir.core.connections import Connections + +from nornir.plugins.connections.napalm import Napalm +from nornir.plugins.connections.netmiko import Netmiko +from nornir.plugins.connections.paramiko import Paramiko + + +def register_default_connection_plugins() -> None: + Connections.register("napalm", Napalm) + Connections.register("netmiko", Netmiko) + Connections.register("paramiko", Paramiko) + + +def InitNornir(config_file="", dry_run=False, configure_logging=True, **kwargs): + """ + Arguments: + config_file(str): Path to the configuration file (optional) + dry_run(bool): Whether to simulate changes or not + **kwargs: Extra information to pass to the + :obj:`nornir.core.configuration.Config` object + + Returns: + :obj:`nornir.core.Nornir`: fully instantiated and configured + """ + register_default_connection_plugins() + + conf = Config(path=config_file, **kwargs) + GlobalState.dry_run = dry_run + + if configure_logging: + conf.logging.configure() + + inv_class = conf.inventory.get_plugin() + transform_function = conf.inventory.get_transform_function() + inv = inv_class.deserialize( + transform_function=transform_function, config=conf, **conf.inventory.options + ) + + return Nornir(inventory=inv, _config=conf) diff --git a/nornir/plugins/connections/__init__.py b/nornir/plugins/connections/__init__.py index 220748f1..e69de29b 100644 --- a/nornir/plugins/connections/__init__.py +++ b/nornir/plugins/connections/__init__.py @@ -1,13 +0,0 @@ -from .napalm import Napalm -from .netmiko import Netmiko -from .paramiko import Paramiko -from nornir.core.connections import Connections - - -def register_default_connection_plugins() -> None: - Connections.register("napalm", Napalm) - Connections.register("netmiko", Netmiko) - Connections.register("paramiko", Paramiko) - - -register_default_connection_plugins() diff --git a/tests/conftest.py b/tests/conftest.py index 23fed3fd..ee9618d5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,7 @@ import os import subprocess -from nornir.core import InitNornir +from nornir import InitNornir from nornir.core.state import GlobalState import pytest diff --git a/tests/core/test_InitNornir.py b/tests/core/test_InitNornir.py index cce21086..42e05c79 100644 --- a/tests/core/test_InitNornir.py +++ b/tests/core/test_InitNornir.py @@ -1,7 +1,7 @@ import os -from nornir.core import InitNornir +from nornir import InitNornir from nornir.core.deserializer.inventory import Inventory diff --git a/tests/core/test_connections.py b/tests/core/test_connections.py index 100c6eb5..2766315a 100644 --- a/tests/core/test_connections.py +++ b/tests/core/test_connections.py @@ -8,7 +8,7 @@ ConnectionPluginAlreadyRegistered, ConnectionPluginNotRegistered, ) -from nornir.plugins.connections import register_default_connection_plugins +from nornir.init_nornir import register_default_connection_plugins import pytest diff --git a/tests/plugins/tasks/networking/test_tcp_ping.py b/tests/plugins/tasks/networking/test_tcp_ping.py index 376d4864..2e15dda0 100644 --- a/tests/plugins/tasks/networking/test_tcp_ping.py +++ b/tests/plugins/tasks/networking/test_tcp_ping.py @@ -1,7 +1,7 @@ import os -from nornir.core import InitNornir +from nornir import InitNornir from nornir.plugins.tasks import networking diff --git a/tox.ini b/tox.ini index 4a7ea34e..3a366d52 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36,py37 +envlist = py36 [testenv] deps = From 67bab3be0ae48bbf1f2787c73f2cd912bdf197b2 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 9 Sep 2018 14:47:00 +0200 Subject: [PATCH 061/109] fix sphinx --- docs/ref/api/nornir.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/ref/api/nornir.rst b/docs/ref/api/nornir.rst index e052a71f..e8d64724 100644 --- a/docs/ref/api/nornir.rst +++ b/docs/ref/api/nornir.rst @@ -4,8 +4,7 @@ Core InitNornir ---------- -.. automethod:: nornir.InitNornir - :module: nornir.init_nornir +.. automethod:: nornir.init_nornir.InitNornir Nornir ------ From 615228ab39f83c32667ca7ce4a463c7ddb684bf4 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sat, 15 Sep 2018 12:18:09 +0200 Subject: [PATCH 062/109] mypy fixes --- nornir/core/connections.py | 2 +- nornir/core/inventory.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nornir/core/connections.py b/nornir/core/connections.py index 5e26397e..fa5b19c6 100644 --- a/nornir/core/connections.py +++ b/nornir/core/connections.py @@ -34,7 +34,7 @@ def open( password: Optional[str], port: Optional[int], platform: Optional[str], - connection_options: Optional[Dict[str, Any]] = None, + extras: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, ) -> None: """ diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index b3a47097..bc3a53c7 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -2,7 +2,7 @@ from typing import Any, Dict, List, Optional from nornir.core.configuration import Config -from nornir.core.connections import Connections +from nornir.core.connections import Connections, ConnectionPlugin from nornir.core.exceptions import ConnectionAlreadyOpen, ConnectionNotOpen @@ -280,9 +280,9 @@ def get_connection(self, connection: str) -> Any: """ if connection not in self.connections: self.open_connection( - connection, - **self.get_connection_parameters(connection).dict(), + connection=connection, configuration=self.config, + **self.get_connection_parameters(connection).dict(), ) return self.connections[connection].connection @@ -306,7 +306,7 @@ def open_connection( extras: Optional[Dict[str, Any]] = None, configuration: Optional[Config] = None, default_to_host_attributes: bool = True, - ) -> None: + ) -> ConnectionPlugin: """ Open a new connection. From 4cf47f8c27ceef22209c1a51c953dd5a6c066604 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sat, 15 Sep 2018 14:42:22 +0200 Subject: [PATCH 063/109] fix attributes resolution --- nornir/core/deserializer/inventory.py | 4 +-- nornir/core/inventory.py | 2 +- tests/core/test_connections.py | 2 +- tests/core/test_filter.py | 18 ++++++++--- tests/core/test_inventory.py | 13 ++++++-- tests/inventory_data/containers.sh | 13 ++++---- tests/inventory_data/groups.yaml | 22 +++++++++++-- tests/inventory_data/hosts.yaml | 32 ++++++++++++++++++- .../text/output_data/basic_inventory.stdout | 4 +++ .../output_data/failed_with_severity.stdout | 8 +++++ .../text/output_data/multiple_tasks.stdout | 7 ++++ 11 files changed, 105 insertions(+), 20 deletions(-) diff --git a/nornir/core/deserializer/inventory.py b/nornir/core/deserializer/inventory.py index a3a93c7b..8199b83d 100644 --- a/nornir/core/deserializer/inventory.py +++ b/nornir/core/deserializer/inventory.py @@ -123,9 +123,7 @@ def deserialize(cls, transform_function=None, *args, **kwargs): groups = inventory.Groups() for n, g in deserialized.groups.items(): - groups[n] = InventoryElement.deserialize_group( - defaults=defaults, name=n, **g.dict() - ) + groups[n] = InventoryElement.deserialize_group(name=n, **g.dict()) return inventory.Inventory( hosts=hosts, diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index bc3a53c7..365230c7 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -178,7 +178,7 @@ def __getattribute__(self, name): v = object.__getattribute__(self, name) if v is None: for g in self.groups.refs: - r = object.__getattribute__(self, name) + r = getattr(g, name) if r is not None: return r diff --git a/tests/core/test_connections.py b/tests/core/test_connections.py index 2766315a..3a0f58ac 100644 --- a/tests/core/test_connections.py +++ b/tests/core/test_connections.py @@ -114,7 +114,7 @@ def test_validate_params_simple(self, nornir): params = { "hostname": "127.0.0.1", "username": "root", - "password": "docker", + "password": "from_group1", "port": 65002, "platform": "junos", "extras": {}, diff --git a/tests/core/test_filter.py b/tests/core/test_filter.py index ad58b93c..a3524bc0 100644 --- a/tests/core/test_filter.py +++ b/tests/core/test_filter.py @@ -41,7 +41,7 @@ def test_negate(self, nornir): f = ~F(groups__contains="group_1") filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) - assert filtered == ["dev3.group_2", "dev4.group_2"] + assert filtered == ["dev3.group_2", "dev4.group_2", "dev5.no_group"] def test_negate_and_second_negate(self, nornir): f = F(site="site1") & ~F(role="www") @@ -53,7 +53,12 @@ def test_negate_or_both_negate(self, nornir): f = ~F(site="site1") | ~F(role="www") filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) - assert filtered == ["dev2.group_1", "dev3.group_2", "dev4.group_2"] + assert filtered == [ + "dev2.group_1", + "dev3.group_2", + "dev4.group_2", + "dev5.no_group", + ] def test_nested_data_a_string(self, nornir): f = F(nested_data__a_string="asdasd") @@ -83,7 +88,12 @@ def test_nested_data_a_dict_doesnt_contain(self, nornir): f = ~F(nested_data__a_dict__contains="a") filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) - assert filtered == ["dev2.group_1", "dev3.group_2", "dev4.group_2"] + assert filtered == [ + "dev2.group_1", + "dev3.group_2", + "dev4.group_2", + "dev5.no_group", + ] def test_nested_data_a_list_contains(self, nornir): f = F(nested_data__a_list__contains=2) @@ -107,7 +117,7 @@ def test_filtering_string_in_list(self, nornir): f = F(platform__in=["linux", "mock"]) filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) - assert filtered == ["dev3.group_2", "dev4.group_2"] + assert filtered == ["dev3.group_2", "dev4.group_2", "dev5.no_group"] def test_filtering_list_any(self, nornir): f = F(nested_data__a_list__any=[1, 3]) diff --git a/tests/core/test_inventory.py b/tests/core/test_inventory.py index 2b5ba88b..de6032cc 100644 --- a/tests/core/test_inventory.py +++ b/tests/core/test_inventory.py @@ -81,6 +81,7 @@ def test_filtering(self): "dev2.group_1", "dev3.group_2", "dev4.group_2", + "dev5.no_group", ] www = sorted(list(inv.filter(role="www").hosts.keys())) @@ -128,8 +129,13 @@ def test_var_resolution(self): inv.hosts["dev3.group_2"].data["my_var"] assert inv.hosts["dev4.group_2"].data["my_var"] == "comes_from_dev4.group_2" + def test_attributes_resolution(self): + inv = deserializer.Inventory.deserialize(**inv_dict) assert inv.hosts["dev1.group_1"].password == "a_password" - assert inv.hosts["dev2.group_1"].password == "docker" + assert inv.hosts["dev2.group_1"].password == "from_group1" + assert inv.hosts["dev3.group_2"].password == "docker" + assert inv.hosts["dev4.group_2"].password == "from_parent_group" + assert inv.hosts["dev5.no_group"].password == "docker" def test_has_parents(self): inv = deserializer.Inventory.deserialize(**inv_dict) @@ -187,4 +193,7 @@ def test_defaults(self): inv = deserializer.Inventory.deserialize(**inv_dict) inv.defaults.password = "asd" assert inv.defaults.password == "asd" - assert inv.hosts["dev2.group_1"].password == "asd" + assert inv.hosts["dev2.group_1"].password == "from_group1" + assert inv.hosts["dev3.group_2"].password == "asd" + assert inv.hosts["dev4.group_2"].password == "from_parent_group" + assert inv.hosts["dev5.no_group"].password == "asd" diff --git a/tests/inventory_data/containers.sh b/tests/inventory_data/containers.sh index 0498be87..f4fad50f 100755 --- a/tests/inventory_data/containers.sh +++ b/tests/inventory_data/containers.sh @@ -1,12 +1,12 @@ #!/bin/bash start () { - docker run -d -p 65001:22 --name dev1.group_1 --hostname=dev1.group_1 dbarroso/stupid_ssh_container - docker run -d -p 65002:22 --name dev2.group_1 --hostname=dev2.group_1 dbarroso/stupid_ssh_container - docker run -d -p 65003:22 --name dev3.group_2 --hostname=dev3.group_2 dbarroso/stupid_ssh_container - docker run -d -p 65004:22 --name dev4.group_2 --hostname=dev4.group_2 dbarroso/stupid_ssh_container - docker run -d -p 65080:80 --name httpbin bungoume/httpbin-container - sleep 3 + docker run -d -p 65001:22 --rm --name dev1.group_1 --hostname=dev1.group_1 dbarroso/stupid_ssh_container + docker run -d -p 65002:22 --rm --name dev2.group_1 --hostname=dev2.group_1 dbarroso/stupid_ssh_container + docker run -d -p 65003:22 --rm --name dev3.group_2 --hostname=dev3.group_2 dbarroso/stupid_ssh_container + docker run -d -p 65004:22 --rm --name dev4.group_2 --hostname=dev4.group_2 dbarroso/stupid_ssh_container + docker run -d -p 65005:22 --rm --name dev5.no_group --hostname=dev5.no_group dbarroso/stupid_ssh_container + docker run -d -p 65080:80 --rm --name httpbin bungoume/httpbin-container } stop () { @@ -14,6 +14,7 @@ stop () { docker rm -f dev2.group_1 docker rm -f dev3.group_2 docker rm -f dev4.group_2 + docker rm -f dev5.no_group docker rm -f httpbin } diff --git a/tests/inventory_data/groups.yaml b/tests/inventory_data/groups.yaml index 390f9c93..5196b6f5 100644 --- a/tests/inventory_data/groups.yaml +++ b/tests/inventory_data/groups.yaml @@ -3,7 +3,7 @@ parent_group: port: hostname: username: - password: + password: from_parent_group platform: data: a_var: blah @@ -17,11 +17,19 @@ parent_group: platform: extras: blah: from_group + dummy2: + hostname: dummy2_from_parent_group + port: + username: + password: + platform: + extras: + blah: from_group group_1: port: hostname: username: - password: + password: from_group1 platform: data: my_var: comes_from_group_1 @@ -39,3 +47,13 @@ group_2: site: site2 groups: [] connection_options: {} +group_3: + port: + hostname: + username: + password: + platform: + groups: [] + data: + site: site2 + connection_options: {} diff --git a/tests/inventory_data/hosts.yaml b/tests/inventory_data/hosts.yaml index 905ed2f0..6a5e9f94 100644 --- a/tests/inventory_data/hosts.yaml +++ b/tests/inventory_data/hosts.yaml @@ -49,7 +49,14 @@ dev2.group_1: a_string: qwe groups: - group_1 - connection_options: {} + connection_options: + paramiko: + port: 65002 + hostname: 127.0.0.1 + username: root + password: docker + platform: linux + extras: {} dev3.group_2: port: 65003 hostname: @@ -81,4 +88,27 @@ dev4.group_2: groups: - group_2 - parent_group + connection_options: + paramiko: + port: 65004 + hostname: 127.0.0.1 + username: root + password: docker + platform: linux + extras: {} + netmiko: + port: 65004 + hostname: 127.0.0.1 + username: root + password: docker + platform: linux + extras: {} +dev5.no_group: + port: 65005 + hostname: localhost + username: + password: + platform: linux + data: {} + groups: [] connection_options: {} diff --git a/tests/plugins/functions/text/output_data/basic_inventory.stdout b/tests/plugins/functions/text/output_data/basic_inventory.stdout index e53e05bf..3b61499e 100644 --- a/tests/plugins/functions/text/output_data/basic_inventory.stdout +++ b/tests/plugins/functions/text/output_data/basic_inventory.stdout @@ -15,3 +15,7 @@ Hello from Nornir vvvv echo_task ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO Hello from Nornir ^^^^ END echo_task ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +* dev5.no_group ** changed : False ********************************************* +vvvv echo_task ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO +Hello from Nornir +^^^^ END echo_task ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/tests/plugins/functions/text/output_data/failed_with_severity.stdout b/tests/plugins/functions/text/output_data/failed_with_severity.stdout index a80685c3..391824ef 100644 --- a/tests/plugins/functions/text/output_data/failed_with_severity.stdout +++ b/tests/plugins/functions/text/output_data/failed_with_severity.stdout @@ -19,3 +19,11 @@ Hello from CRITICAL ---- parse_data ** changed : False --------------------------------------------- ERROR Exception('Unknown Error -> Contact your system administrator',) ^^^^ END read_data ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +* dev5.no_group ** changed : False ********************************************* +vvvv read_data ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv ERROR +NornirSubTaskError() +---- echo_task ** changed : False ---------------------------------------------- CRITICAL +Hello from CRITICAL +---- parse_data ** changed : False --------------------------------------------- ERROR +KeyError('values',) +^^^^ END read_data ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/tests/plugins/functions/text/output_data/multiple_tasks.stdout b/tests/plugins/functions/text/output_data/multiple_tasks.stdout index f75e1843..ba555039 100644 --- a/tests/plugins/functions/text/output_data/multiple_tasks.stdout +++ b/tests/plugins/functions/text/output_data/multiple_tasks.stdout @@ -28,3 +28,10 @@ Hello from Nornir ---- load_data ** changed : False ---------------------------------------------- INFO {'os': 'Linux', 'services': ['http', 'smtp', 'dns']} ^^^^ END data_with_greeting ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +* dev5.no_group ** changed : False ********************************************* +vvvv data_with_greeting ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO +---- echo_task ** changed : False ---------------------------------------------- INFO +Hello from Nornir +---- load_data ** changed : False ---------------------------------------------- INFO +{'os': 'Linux', 'services': ['http', 'smtp', 'dns']} +^^^^ END data_with_greeting ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From e3feae4108d8c85589606a99061c96a54b305ef0 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sat, 15 Sep 2018 14:55:00 +0200 Subject: [PATCH 064/109] fix bug in tutorial --- docs/howto/inventory.ipynb | 165 ++++++++++++++++++++++++++++++------- 1 file changed, 134 insertions(+), 31 deletions(-) diff --git a/docs/howto/inventory.ipynb b/docs/howto/inventory.ipynb index 39993fff..f28c3e3e 100644 --- a/docs/howto/inventory.ipynb +++ b/docs/howto/inventory.ipynb @@ -926,6 +926,109 @@ "%highlight_file inventory/groups.yaml" ] }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
1 ---\n",
+       "2 username: admin\n",
+       "3 data:\n",
+       "4     domain: acme.local\n",
+       "
\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# defaults file\n", + "%highlight_file inventory/defaults.yaml" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -937,16 +1040,16 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'acme.local'" + "'global.local'" ] }, - "execution_count": 13, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -958,7 +1061,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -967,7 +1070,7 @@ "65100" ] }, - "execution_count": 14, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -985,7 +1088,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -994,7 +1097,7 @@ "'acme.local'" ] }, - "execution_count": 15, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -1013,7 +1116,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -1040,7 +1143,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -1071,7 +1174,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -1080,7 +1183,7 @@ "dict_keys(['host1.cmh', 'host2.cmh', 'spine00.cmh', 'spine01.cmh', 'leaf00.cmh', 'leaf01.cmh'])" ] }, - "execution_count": 18, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -1098,7 +1201,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -1107,7 +1210,7 @@ "dict_keys(['spine00.cmh', 'spine01.cmh'])" ] }, - "execution_count": 19, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -1125,7 +1228,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -1134,7 +1237,7 @@ "dict_keys(['spine00.cmh', 'spine01.cmh'])" ] }, - "execution_count": 20, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -1152,7 +1255,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -1161,7 +1264,7 @@ "dict_keys(['spine00.cmh', 'spine01.cmh'])" ] }, - "execution_count": 21, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -1173,7 +1276,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 23, "metadata": {}, "outputs": [ { @@ -1182,7 +1285,7 @@ "dict_keys(['leaf00.cmh', 'leaf01.cmh'])" ] }, - "execution_count": 22, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } @@ -1216,7 +1319,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 24, "metadata": {}, "outputs": [ { @@ -1225,7 +1328,7 @@ "dict_keys(['spine00.cmh', 'spine01.cmh', 'spine00.bma', 'spine01.bma'])" ] }, - "execution_count": 23, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -1239,7 +1342,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 25, "metadata": {}, "outputs": [ { @@ -1248,7 +1351,7 @@ "dict_keys(['host1.cmh', 'host2.cmh', 'host1.bma', 'host2.bma'])" ] }, - "execution_count": 24, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -1269,7 +1372,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -1279,7 +1382,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -1298,7 +1401,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -1317,7 +1420,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 29, "metadata": {}, "outputs": [ { @@ -1336,7 +1439,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -1362,7 +1465,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 31, "metadata": {}, "outputs": [ { @@ -1380,7 +1483,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 32, "metadata": {}, "outputs": [ { @@ -1398,7 +1501,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 33, "metadata": {}, "outputs": [ { From d106aec7868c7e604ab11b9835b06e592c5ccd18 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Fri, 21 Sep 2018 15:54:29 +0200 Subject: [PATCH 065/109] minor bugfix --- nornir/core/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nornir/core/task.py b/nornir/core/task.py index fd266026..ee190343 100644 --- a/nornir/core/task.py +++ b/nornir/core/task.py @@ -117,7 +117,7 @@ def is_dry_run(self, override=None): Arguments: override (bool): Override for current task """ - return override if override is not None else self.nornir.dry_run + return override if override is not None else self.nornir.data.dry_run class Result(object): From 1afe62c241c1a1d016761e58fadedd7e83e3915f Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 23 Sep 2018 07:47:56 -0600 Subject: [PATCH 066/109] inhering from host atrributes --- nornir/core/inventory.py | 9 ++++++++- setup.cfg | 2 +- tests/core/test_connections.py | 8 ++++---- tests/core/test_inventory.py | 24 ++++++++++++------------ 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 365230c7..de8bb6f7 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -235,7 +235,14 @@ def get_connection_parameters( else: r = self._get_connection_options_recursively(connection) if r is not None: - return r + d = ConnectionOptions( + hostname=r.hostname if r.hostname is not None else self.hostname, + port=r.port if r.port is not None else self.port, + username=r.username if r.username is not None else self.username, + password=r.password if r.password is not None else self.password, + platform=r.platform if r.platform is not None else self.platform, + extras=r.extras if r.extras is not None else {}, + ) else: d = ConnectionOptions( hostname=self.hostname, diff --git a/setup.cfg b/setup.cfg index 6e4d920b..14f458eb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,7 @@ max-complexity = 10 max-line-length = 100 [tool:pytest] -addopts = --cov=nornir --cov-report=term-missing -vs +#addopts = --cov=nornir --cov-report=term-missing -vs python_paths = ./ [mypy] diff --git a/tests/core/test_connections.py b/tests/core/test_connections.py index 3a0f58ac..2a02f259 100644 --- a/tests/core/test_connections.py +++ b/tests/core/test_connections.py @@ -131,11 +131,11 @@ def test_validate_params_simple(self, nornir): def test_validate_params_overrides(self, nornir): params = { - "port": None, + "port": 65002, "hostname": "dummy_from_parent_group", - "username": None, - "password": None, - "platform": None, + "username": "root", + "password": "from_group1", + "platform": "junos", "extras": {"blah": "from_group"}, } nr = nornir.filter(name="dev2.group_1") diff --git a/tests/core/test_inventory.py b/tests/core/test_inventory.py index de6032cc..9374eb99 100644 --- a/tests/core/test_inventory.py +++ b/tests/core/test_inventory.py @@ -154,11 +154,11 @@ def test_get_connection_parameters(self): inv = deserializer.Inventory.deserialize(**inv_dict) p1 = inv.hosts["dev1.group_1"].get_connection_parameters("dummy") assert p1.dict() == { - "port": None, + "port": 65001, "hostname": "dummy_from_host", - "username": None, - "password": None, - "platform": None, + "username": "root", + "password": "a_password", + "platform": "eos", "extras": {"blah": "from_host"}, } p2 = inv.hosts["dev1.group_1"].get_connection_parameters("asd") @@ -172,20 +172,20 @@ def test_get_connection_parameters(self): } p3 = inv.hosts["dev2.group_1"].get_connection_parameters("dummy") assert p3.dict() == { - "port": None, + "port": 65002, "hostname": "dummy_from_parent_group", - "username": None, - "password": None, - "platform": None, + "username": "root", + "password": "from_group1", + "platform": "junos", "extras": {"blah": "from_group"}, } p4 = inv.hosts["dev3.group_2"].get_connection_parameters("dummy") assert p4.dict() == { - "port": None, + "port": 65003, "hostname": "dummy_from_defaults", - "username": None, - "password": None, - "platform": None, + "username": "root", + "password": "docker", + "platform": "linux", "extras": {"blah": "from_defaults"}, } From 37a6db1ade4d7bf6f15a8e1867fb977d67fb36e1 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 23 Sep 2018 08:11:47 -0600 Subject: [PATCH 067/109] resolve connection_options recursively --- nornir/core/deserializer/inventory.py | 2 +- nornir/core/inventory.py | 29 +++++++++++++++++++-------- setup.cfg | 2 +- tests/core/test_connections.py | 15 ++++++++++++++ tests/inventory_data/hosts.yaml | 7 +++++++ 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/nornir/core/deserializer/inventory.py b/nornir/core/deserializer/inventory.py index 8199b83d..a9ebccad 100644 --- a/nornir/core/deserializer/inventory.py +++ b/nornir/core/deserializer/inventory.py @@ -22,7 +22,7 @@ class Config: class ConnectionOptions(BaseAttributes): - extras: Dict[str, Any] = {} + extras: Optional[Dict[str, Any]] class InventoryElement(BaseAttributes): diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index de8bb6f7..11e0ad7a 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -40,7 +40,7 @@ class ConnectionOptions(BaseAttributes): __slots__ = ("extras",) def __init__(self, extras: Optional[Dict[str, Any]] = None, **kwargs) -> None: - self.extras = extras or {} + self.extras = extras super().__init__(**kwargs) @@ -259,14 +259,27 @@ def _get_connection_options_recursively( ) -> Optional[ConnectionOptions]: p = self.connection_options.get(connection) if p is None: - for g in self.groups.refs: - p = g._get_connection_options_recursively(connection) - if p is not None: - return p + p = ConnectionOptions() - return self.defaults.connection_options.get(connection, None) - else: - return p + for g in self.groups.refs: + sp = g._get_connection_options_recursively(connection) + if sp is not None: + p.hostname = p.hostname if p.hostname is not None else sp.hostname + p.port = p.port if p.port is not None else sp.port + p.username = p.username if p.username is not None else sp.username + p.password = p.password if p.password is not None else sp.password + p.platform = p.platform if p.platform is not None else sp.platform + p.extras = p.extras if p.extras is not None else sp.extras + + sp = self.defaults.connection_options.get(connection, None) + if sp is not None: + p.hostname = p.hostname if p.hostname is not None else sp.hostname + p.port = p.port if p.port is not None else sp.port + p.username = p.username if p.username is not None else sp.username + p.password = p.password if p.password is not None else sp.password + p.platform = p.platform if p.platform is not None else sp.platform + p.extras = p.extras if p.extras is not None else sp.extras + return p def get_connection(self, connection: str) -> Any: """ diff --git a/setup.cfg b/setup.cfg index 14f458eb..6e4d920b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,7 @@ max-complexity = 10 max-line-length = 100 [tool:pytest] -#addopts = --cov=nornir --cov-report=term-missing -vs +addopts = --cov=nornir --cov-report=term-missing -vs python_paths = ./ [mypy] diff --git a/tests/core/test_connections.py b/tests/core/test_connections.py index 2a02f259..10ba1f75 100644 --- a/tests/core/test_connections.py +++ b/tests/core/test_connections.py @@ -84,6 +84,7 @@ class Test(object): def setup_class(cls): Connections.deregister_all() Connections.register("dummy", DummyConnectionPlugin) + Connections.register("dummy2", DummyConnectionPlugin) Connections.register("dummy_no_overrides", DummyConnectionPlugin) def test_open_and_close_connection(self, nornir): @@ -143,6 +144,20 @@ def test_validate_params_overrides(self, nornir): assert len(r) == 1 assert not r.failed + def test_validate_params_overrides_groups(self, nornir): + params = { + "port": 65002, + "hostname": "dummy2_from_parent_group", + "username": "dummy2_from_host", + "password": "from_group1", + "platform": "junos", + "extras": {"blah": "from_group"}, + } + nr = nornir.filter(name="dev2.group_1") + r = nr.run(task=validate_params, conn="dummy2", params=params, num_workers=1) + assert len(r) == 1 + assert not r.failed + class TestConnectionPluginsRegistration(object): def setup_method(self, method): diff --git a/tests/inventory_data/hosts.yaml b/tests/inventory_data/hosts.yaml index 6a5e9f94..9924a506 100644 --- a/tests/inventory_data/hosts.yaml +++ b/tests/inventory_data/hosts.yaml @@ -57,6 +57,13 @@ dev2.group_1: password: docker platform: linux extras: {} + dummy2: + hostname: + port: + username: dummy2_from_host + password: + platform: + extras: dev3.group_2: port: 65003 hostname: From 0c0dd5fd0ff62660e3c4a48c8640943745068ecf Mon Sep 17 00:00:00 2001 From: David Barroso Date: Mon, 24 Sep 2018 14:05:52 -0600 Subject: [PATCH 068/109] fix nbval --- docs/howto/inventory.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/howto/inventory.ipynb b/docs/howto/inventory.ipynb index f28c3e3e..3dc7f693 100644 --- a/docs/howto/inventory.ipynb +++ b/docs/howto/inventory.ipynb @@ -380,7 +380,6 @@ " \"extras\": {\n", " \"title\": \"Extras\",\n", " \"required\": false,\n", - " \"default\": {},\n", " \"type\": \"mapping\",\n", " \"item_type\": \"any\",\n", " \"key_type\": \"str\"\n", From 55e129f3714953eaa537d576a1b0d5db8dd9605e Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Tue, 2 Oct 2018 01:46:10 -0700 Subject: [PATCH 069/109] Always go into enable mode for config changes (#251) --- nornir/plugins/tasks/networking/netmiko_send_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nornir/plugins/tasks/networking/netmiko_send_config.py b/nornir/plugins/tasks/networking/netmiko_send_config.py index 56633f69..0d423332 100644 --- a/nornir/plugins/tasks/networking/netmiko_send_config.py +++ b/nornir/plugins/tasks/networking/netmiko_send_config.py @@ -22,6 +22,7 @@ def netmiko_send_config( * result (``dict``): dictionary showing the CLI from the configuration changes """ net_connect = task.host.get_connection("netmiko") + net_connect.enable() if config_commands: result = net_connect.send_config_set(config_commands=config_commands, **kwargs) elif config_file: From ac82aa03e86be2a9c334798c766eb20868322cb0 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 7 Oct 2018 13:24:39 +0200 Subject: [PATCH 070/109] stop printing coverage by default --- setup.cfg | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 6e4d920b..14f458eb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,7 @@ max-complexity = 10 max-line-length = 100 [tool:pytest] -addopts = --cov=nornir --cov-report=term-missing -vs +#addopts = --cov=nornir --cov-report=term-missing -vs python_paths = ./ [mypy] diff --git a/tox.ini b/tox.ini index 3a366d52..f5a712d2 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ deps = passenv = * commands = - py.test + py.test --cov=nornir --cov-report=term-missing -vs [testenv:black] deps = black==18.6b4 From 4d9d839f1df725f32700a3deed3c992d73fe8c2d Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 7 Oct 2018 17:02:24 +0200 Subject: [PATCH 071/109] separate serializing function for the configuration --- nornir/core/configuration.py | 150 ++++++--------- nornir/core/deserializer/configuration.py | 150 +++++++++++++++ nornir/core/deserializer/inventory.py | 4 +- nornir/core/inventory.py | 11 +- nornir/init_nornir.py | 13 +- .../empty.yaml => deserializer/__init__.py} | 0 tests/core/deserializer/my_jinja_filters.py | 10 + tests/core/deserializer/test_configuration.py | 172 ++++++++++++++++++ .../test_configuration/config.yaml | 2 +- .../test_configuration/empty.yaml | 0 tests/core/test_configuration.py | 60 ------ 11 files changed, 408 insertions(+), 164 deletions(-) create mode 100644 nornir/core/deserializer/configuration.py rename tests/core/{test_configuration/empty.yaml => deserializer/__init__.py} (100%) create mode 100644 tests/core/deserializer/my_jinja_filters.py create mode 100644 tests/core/deserializer/test_configuration.py rename tests/core/{ => deserializer}/test_configuration/config.yaml (56%) create mode 100644 tests/core/deserializer/test_configuration/empty.yaml delete mode 100644 tests/core/test_configuration.py diff --git a/nornir/core/configuration.py b/nornir/core/configuration.py index 55c4b32d..5e53e5a0 100644 --- a/nornir/core/configuration.py +++ b/nornir/core/configuration.py @@ -1,60 +1,44 @@ -import importlib import logging import logging.config -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Type, TYPE_CHECKING -from pydantic import BaseSettings -import ruamel.yaml +if TYPE_CHECKING: + from nornir.core.inventory import Inventory # noqa -class SSHConfig(BaseSettings): - """ - Args: - config_file: User ssh_config_file - """ +class SSHConfig(object): + __slots__ = "config_file" - config_file: str = "~/.ssh/config" + def __init__(self, config_file: str) -> None: + self.config_file = config_file - class Config: - env_prefix = "NORNIR_SSH_" - ignore_extra = False +class InventoryConfig(object): + __slots__ = "plugin", "options", "transform_function" -class Inventory(BaseSettings): - """ - Args: - plugin: Path to inventory modules. - transform_function: Path to transform function. The transform_function you provide - will run against each host in the inventory - options: Arguments to pass to the inventory plugin - """ + def __init__( + self, + plugin: Type["Inventory"], + options: Dict[str, Any], + transform_function: Optional[Callable[..., Any]], + ) -> None: + self.plugin = plugin + self.options = options + self.transform_function = transform_function - plugin: Any = "nornir.plugins.inventory.simple.SimpleInventory" - options: Dict[str, Any] = {} - transform_function: Any = "" - def get_plugin(self) -> Optional[Callable[..., Any]]: - return _resolve_import_from_string(self.plugin) +class LoggingConfig(object): + __slots__ = "level", "file", "format", "to_console", "loggers" - def get_transform_function(self) -> Optional[Callable[..., Any]]: - return _resolve_import_from_string(self.transform_function) - - class Config: - env_prefix = "NORNIR_INVENTORY_" - ignore_extra = False - - -class Logging(BaseSettings): - level: str = "debug" - file: str = "nornir.log" - format: str = "%(asctime)s - %(name)12s - %(levelname)8s - %(funcName)10s() - %(message)s" - to_console: bool = False - loggers: List[str] = ["nornir"] - - class Config: - env_prefix = "NORNIR_LOGGING_" - ignore_extra = False + def __init__( + self, level: int, file_: str, format_: str, to_console: bool, loggers: List[str] + ) -> None: + self.level = level + self.file = file_ + self.format = format_ + self.to_console = to_console + self.loggers = loggers def configure(self): rootHandlers: List[Any] = [] @@ -97,57 +81,37 @@ def configure(self): } for logger in self.loggers: - loggers[logger] = {"level": self.level.upper(), "handlers": handlers_list} + loggers[logger] = {"level": self.level, "handlers": handlers_list} if rootHandlers: logging.config.dictConfig(dictConfig) -class Config(BaseSettings): - """ - Args: - inventory: Dictionary with Inventory options - jinja_filters: Path to callable returning jinja filters to be used - raise_on_error: If set to ``True``, (:obj:`nornir.core.Nornir.run`) method of - will raise an exception if at least a host failed - num_workers: Number of Nornir worker processes that are run at the same time - configuration can be overridden on individual tasks by using the - - - """ - - inventory: Inventory = Inventory() - jinja_filters: str = "" - num_workers: int = 20 - raise_on_error: bool = False - ssh: SSHConfig = SSHConfig() - user_defined: Dict[str, Any] = {} - logging: Logging = Logging() - - class Config: - env_prefix = "NORNIR_" - ignore_extra = False - - def __init__(self, path: str = "", **kwargs) -> None: - if path: - with open(path, "r") as f: - yml = ruamel.yaml.YAML(typ="safe") - data = yml.load(f) or {} - data.update(kwargs) - else: - data = kwargs - data["ssh"] = SSHConfig(**data.pop("ssh", {})) - data["inventory"] = Inventory(**data.pop("inventory", {})) - data["logging"] = Logging(**data.pop("logging", {})) - super().__init__(**data) - - -def _resolve_import_from_string(import_path: Any) -> Optional[Callable[..., Any]]: - if not import_path: - return None - elif callable(import_path): - return import_path - module_name = ".".join(import_path.split(".")[:-1]) - obj_name = import_path.split(".")[-1] - module = importlib.import_module(module_name) - return getattr(module, obj_name) +class Config(object): + __slots__ = ( + "inventory", + "jinja_filters", + "num_workers", + "raise_on_error", + "ssh", + "user_defined", + "logging", + ) + + def __init__( + self, + inventory: InventoryConfig, + ssh: SSHConfig, + logging: LoggingConfig, + jinja_filters: Optional[Dict[str, Callable[..., Any]]], + num_workers: int, + raise_on_error: bool, + user_defined: Dict[str, Any], + ) -> None: + self.inventory = inventory + self.ssh = ssh + self.logging = logging + self.jinja_filters = jinja_filters or {} + self.num_workers = num_workers + self.raise_on_error = raise_on_error + self.user_defined = user_defined diff --git a/nornir/core/deserializer/configuration.py b/nornir/core/deserializer/configuration.py new file mode 100644 index 00000000..03bf3248 --- /dev/null +++ b/nornir/core/deserializer/configuration.py @@ -0,0 +1,150 @@ +from typing import Any, Callable, Dict, List, Optional + +import importlib +import logging + +from nornir.core import configuration + +from pydantic import BaseSettings + +import ruamel.yaml + + +logger = logging.getLogger("nornir") + + +class SSHConfig(BaseSettings): + """ + Args: + config_file: User ssh_config_file + """ + + config_file: str = "~/.ssh/config" + + class Config: + env_prefix = "NORNIR_SSH_" + ignore_extra = False + + @classmethod + def deserialize(self, **kwargs) -> configuration.SSHConfig: + s = SSHConfig(**kwargs) + return configuration.SSHConfig(config_file=s.config_file) + + +class InventoryConfig(BaseSettings): + """ + Args: + plugin: Path to inventory modules. + transform_function: Path to transform function. The transform_function you provide + will run against each host in the inventory + options: Arguments to pass to the inventory plugin + """ + + plugin: Any = "nornir.plugins.inventory.simple.SimpleInventory" + options: Dict[str, Any] = {} + transform_function: Any = "" + + class Config: + env_prefix = "NORNIR_INVENTORY_" + ignore_extra = False + + @classmethod + def deserialize(self, **kwargs) -> configuration.InventoryConfig: + inv = InventoryConfig(**kwargs) + return configuration.InventoryConfig( + plugin=_resolve_import_from_string(inv.plugin), + options=inv.options, + transform_function=_resolve_import_from_string(inv.transform_function), + ) + + +class LoggingConfig(BaseSettings): + level: str = "debug" + file: str = "nornir.log" + format: str = "%(asctime)s - %(name)12s - %(levelname)8s - %(funcName)10s() - %(message)s" + to_console: bool = False + loggers: List[str] = ["nornir"] + + class Config: + env_prefix = "NORNIR_LOGGING_" + ignore_extra = False + + @classmethod + def deserialize(self, **kwargs) -> configuration.LoggingConfig: + logging_config = LoggingConfig(**kwargs) + return configuration.LoggingConfig( + level=getattr(logging, logging_config.level.upper()), + file_=logging_config.file, + format_=logging_config.format, + to_console=logging_config.to_console, + loggers=logging_config.loggers, + ) + + +class Config(BaseSettings): + """ + Args: + inventory: Dictionary with Inventory options + jinja_filters: Path to callable returning jinja filters to be used + raise_on_error: If set to ``True``, (:obj:`nornir.core.Nornir.run`) method of + will raise an exception if at least a host failed + num_workers: Number of Nornir worker processes that are run at the same time + configuration can be overridden on individual tasks by using the + """ + + inventory: InventoryConfig = InventoryConfig() + ssh: SSHConfig = SSHConfig() + logging: LoggingConfig = LoggingConfig() + jinja_filters: str = "" + num_workers: int = 20 + raise_on_error: bool = False + user_defined: Dict[str, Any] = {} + + class Config: + env_prefix = "NORNIR_" + ignore_extra = False + + @classmethod + def deserialize(cls, **kwargs) -> configuration.Config: + c = Config( + ssh=SSHConfig(**kwargs.pop("ssh", {})), + inventory=InventoryConfig(**kwargs.pop("inventory", {})), + logging=LoggingConfig(**kwargs.pop("logging", {})), + **kwargs, + ) + + jinja_filter_func = _resolve_import_from_string(c.jinja_filters) + jinja_filters = jinja_filter_func() if jinja_filter_func else {} + return configuration.Config( + inventory=InventoryConfig.deserialize(**c.inventory.dict()), + ssh=SSHConfig.deserialize(**c.ssh.dict()), + logging=LoggingConfig.deserialize(**c.logging.dict()), + jinja_filters=jinja_filters, + num_workers=c.num_workers, + raise_on_error=c.raise_on_error, + user_defined=c.user_defined, + ) + + @classmethod + def load_from_file(cls, config_file: str, **kwargs) -> configuration.Config: + config_dict = {} + if config_file: + yml = ruamel.yaml.YAML(typ="safe") + with open(config_file, "r") as f: + config_dict = yml.load(f) or {} + return Config.deserialize(**{**config_dict, **kwargs}) + + +def _resolve_import_from_string(import_path: Any) -> Optional[Callable[..., Any]]: + try: + if not import_path: + return None + elif callable(import_path): + return import_path + module_name = ".".join(import_path.split(".")[:-1]) + obj_name = import_path.split(".")[-1] + module = importlib.import_module(module_name) + return getattr(module, obj_name) + except Exception as e: + logger.error(f"failed to load import_path '{import_path}'\n{e}") + raise diff --git a/nornir/core/deserializer/inventory.py b/nornir/core/deserializer/inventory.py index a9ebccad..7a4f8e56 100644 --- a/nornir/core/deserializer/inventory.py +++ b/nornir/core/deserializer/inventory.py @@ -1,6 +1,7 @@ from typing import Any, Dict, List, Optional, Union from nornir.core import inventory +from nornir.core.deserializer.configuration import Config from pydantic import BaseModel @@ -107,7 +108,7 @@ class Inventory(BaseModel): defaults: Defaults @classmethod - def deserialize(cls, transform_function=None, *args, **kwargs): + def deserialize(cls, config=None, transform_function=None, *args, **kwargs): deserialized = cls(*args, **kwargs) defaults_dict = deserialized.defaults.dict() @@ -130,6 +131,7 @@ def deserialize(cls, transform_function=None, *args, **kwargs): groups=groups, defaults=defaults, transform_function=transform_function, + config=config or Config.deserialize(), ) @classmethod diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 11e0ad7a..4881f47d 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -73,7 +73,7 @@ def __init__( self.groups = groups or ParentGroups() self.data = data or {} self.connection_options = connection_options or {} - self.config = config or Config() + self.config = config super().__init__(**kwargs) @@ -418,7 +418,7 @@ def __init__( for h in self.hosts.values(): transform_function(h) - self.config = config or Config() + self.config = config def filter(self, filter_obj=None, filter_func=None, *args, **kwargs): filter_func = filter_obj or filter_func @@ -430,7 +430,12 @@ def filter(self, filter_obj=None, filter_func=None, *args, **kwargs): for n, h in self.hosts.items() if all(h.get(k) == v for k, v in kwargs.items()) } - return Inventory(hosts=filtered, groups=self.groups, defaults=self.defaults) + return Inventory( + hosts=filtered, + groups=self.groups, + defaults=self.defaults, + config=self.config, + ) def __len__(self): return self.hosts.__len__() diff --git a/nornir/init_nornir.py b/nornir/init_nornir.py index 008b4e1f..1f3beea7 100644 --- a/nornir/init_nornir.py +++ b/nornir/init_nornir.py @@ -1,5 +1,5 @@ from nornir.core import Nornir -from nornir.core.configuration import Config +from nornir.core.deserializer.configuration import Config from nornir.core.state import GlobalState from nornir.core.connections import Connections @@ -27,16 +27,17 @@ def InitNornir(config_file="", dry_run=False, configure_logging=True, **kwargs): """ register_default_connection_plugins() - conf = Config(path=config_file, **kwargs) + conf = Config.load_from_file(config_file, **kwargs) + GlobalState.dry_run = dry_run if configure_logging: conf.logging.configure() - inv_class = conf.inventory.get_plugin() - transform_function = conf.inventory.get_transform_function() - inv = inv_class.deserialize( - transform_function=transform_function, config=conf, **conf.inventory.options + inv = conf.inventory.plugin.deserialize( + transform_function=conf.inventory.transform_function, + config=conf, + **conf.inventory.options ) return Nornir(inventory=inv, _config=conf) diff --git a/tests/core/test_configuration/empty.yaml b/tests/core/deserializer/__init__.py similarity index 100% rename from tests/core/test_configuration/empty.yaml rename to tests/core/deserializer/__init__.py diff --git a/tests/core/deserializer/my_jinja_filters.py b/tests/core/deserializer/my_jinja_filters.py new file mode 100644 index 00000000..554582ae --- /dev/null +++ b/tests/core/deserializer/my_jinja_filters.py @@ -0,0 +1,10 @@ +def upper(blah: str) -> str: + return blah.upper() + + +def lower(blah: str) -> str: + return blah.lower() + + +def jinja_filters(): + return {"upper": upper, "lower": lower} diff --git a/tests/core/deserializer/test_configuration.py b/tests/core/deserializer/test_configuration.py new file mode 100644 index 00000000..c6ce27a9 --- /dev/null +++ b/tests/core/deserializer/test_configuration.py @@ -0,0 +1,172 @@ +import logging +import os + +from nornir.core.configuration import Config +from nornir.plugins.inventory.simple import SimpleInventory +from nornir.plugins.inventory.ansible import AnsibleInventory +from nornir.core.deserializer.configuration import Config as ConfigDeserializer + +from tests.core.deserializer import my_jinja_filters + +import pytest + +dir_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "test_configuration" +) + + +DEFAULT_LOG_FORMAT = ( + "%(asctime)s - %(name)12s - %(levelname)8s - %(funcName)10s() - %(message)s" +) + + +class Test(object): + def test_config_defaults(self): + c = ConfigDeserializer() + assert c.dict() == { + "inventory": { + "plugin": "nornir.plugins.inventory.simple.SimpleInventory", + "options": {}, + "transform_function": "", + }, + "ssh": {"config_file": "~/.ssh/config"}, + "logging": { + "level": "debug", + "file": "nornir.log", + "format": DEFAULT_LOG_FORMAT, + "to_console": False, + "loggers": ["nornir"], + }, + "jinja_filters": "", + "num_workers": 20, + "raise_on_error": False, + "user_defined": {}, + } + + def test_config_basic(self): + c = ConfigDeserializer( + num_workers=30, logging={"file": ""}, user_defined={"my_opt": True} + ) + assert c.dict() == { + "inventory": { + "plugin": "nornir.plugins.inventory.simple.SimpleInventory", + "options": {}, + "transform_function": "", + }, + "ssh": {"config_file": "~/.ssh/config"}, + "logging": { + "level": "debug", + "file": "", + "format": DEFAULT_LOG_FORMAT, + "to_console": False, + "loggers": ["nornir"], + }, + "jinja_filters": "", + "num_workers": 30, + "raise_on_error": False, + "user_defined": {"my_opt": True}, + } + + def test_deserialize_defaults(self): + c = ConfigDeserializer.deserialize() + assert isinstance(c, Config) + + assert c.num_workers == 20 + assert not c.raise_on_error + assert c.user_defined == {} + + assert c.logging.level == logging.DEBUG + assert c.logging.file == "nornir.log" + assert c.logging.format == DEFAULT_LOG_FORMAT + assert not c.logging.to_console + assert c.logging.loggers == ["nornir"] + + assert c.ssh.config_file == "~/.ssh/config" + + assert c.inventory.plugin == SimpleInventory + assert c.inventory.options == {} + assert c.inventory.transform_function is None + + def test_deserialize_basic(self): + c = ConfigDeserializer.deserialize( + num_workers=30, + user_defined={"my_opt": True}, + logging={"file": "", "level": "info"}, + ssh={"config_file": "~/.ssh/alt_config"}, + inventory={"plugin": "nornir.plugins.inventory.ansible.AnsibleInventory"}, + ) + assert isinstance(c, Config) + + assert c.num_workers == 30 + assert not c.raise_on_error + assert c.user_defined == {"my_opt": True} + + assert c.logging.level == logging.INFO + assert c.logging.file == "" + assert c.logging.format == DEFAULT_LOG_FORMAT + assert not c.logging.to_console + assert c.logging.loggers == ["nornir"] + + assert c.ssh.config_file == "~/.ssh/alt_config" + + assert c.inventory.plugin == AnsibleInventory + assert c.inventory.options == {} + assert c.inventory.transform_function is None + + def test_jinja_filters(self): + c = ConfigDeserializer.deserialize( + jinja_filters="tests.core.deserializer.my_jinja_filters.jinja_filters" + ) + assert c.jinja_filters == my_jinja_filters.jinja_filters() + + def test_jinja_filters_error(self): + with pytest.raises(ModuleNotFoundError): + ConfigDeserializer.deserialize(jinja_filters="asdasd.asdasd") + + def test_configuration_file_empty(self): + config = ConfigDeserializer.load_from_file( + os.path.join(dir_path, "empty.yaml"), user_defined={"asd": "qwe"} + ) + assert config.user_defined["asd"] == "qwe" + assert config.num_workers == 20 + assert not config.raise_on_error + assert config.inventory.plugin == SimpleInventory + + def test_configuration_file_normal(self): + config = ConfigDeserializer.load_from_file( + os.path.join(dir_path, "config.yaml") + ) + assert config.num_workers == 10 + assert not config.raise_on_error + assert config.inventory.plugin == AnsibleInventory + + def test_configuration_file_override_argument(self): + config = ConfigDeserializer.load_from_file( + os.path.join(dir_path, "config.yaml"), num_workers=20, raise_on_error=True + ) + assert config.num_workers == 20 + assert config.raise_on_error + + def test_configuration_file_override_env(self): + os.environ["NORNIR_NUM_WORKERS"] = "30" + os.environ["NORNIR_RAISE_ON_ERROR"] = "1" + os.environ["NORNIR_SSH_CONFIG_FILE"] = "/user/ssh_config" + config = ConfigDeserializer.deserialize() + assert config.num_workers == 30 + assert config.raise_on_error + assert config.ssh.config_file == "/user/ssh_config" + os.environ.pop("NORNIR_NUM_WORKERS") + os.environ.pop("NORNIR_RAISE_ON_ERROR") + os.environ.pop("NORNIR_SSH_CONFIG_FILE") + + def test_configuration_bool_env(self): + os.environ["NORNIR_RAISE_ON_ERROR"] = "0" + config = ConfigDeserializer.deserialize() + assert config.num_workers == 20 + assert not config.raise_on_error + + def test_get_user_defined_from_file(self): + config = ConfigDeserializer.load_from_file( + os.path.join(dir_path, "config.yaml") + ) + assert config.user_defined["asd"] == "qwe" diff --git a/tests/core/test_configuration/config.yaml b/tests/core/deserializer/test_configuration/config.yaml similarity index 56% rename from tests/core/test_configuration/config.yaml rename to tests/core/deserializer/test_configuration/config.yaml index bf88d73b..ebd6f387 100644 --- a/tests/core/test_configuration/config.yaml +++ b/tests/core/deserializer/test_configuration/config.yaml @@ -2,6 +2,6 @@ num_workers: 10 raise_on_error: false inventory: - plugin: something + plugin: nornir.plugins.inventory.ansible.AnsibleInventory user_defined: asd: qwe diff --git a/tests/core/deserializer/test_configuration/empty.yaml b/tests/core/deserializer/test_configuration/empty.yaml new file mode 100644 index 00000000..e69de29b diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py deleted file mode 100644 index df59501a..00000000 --- a/tests/core/test_configuration.py +++ /dev/null @@ -1,60 +0,0 @@ -import os - -from nornir.core.configuration import Config -from nornir.plugins.inventory.simple import SimpleInventory - -# import pytest - - -dir_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "test_configuration" -) - - -class Test(object): - def test_configuration_empty(self): - config = Config( - os.path.join(dir_path, "empty.yaml"), user_defined={"asd": "qwe"} - ) - assert config.user_defined["asd"] == "qwe" - assert config.num_workers == 20 - assert not config.raise_on_error - assert ( - config.inventory.plugin == "nornir.plugins.inventory.simple.SimpleInventory" - ) - assert config.inventory.get_plugin() == SimpleInventory - - def test_configuration_normal(self): - config = Config(os.path.join(dir_path, "config.yaml")) - assert config.num_workers == 10 - assert not config.raise_on_error - assert config.inventory.plugin == "something" - - def test_configuration_normal_override_argument(self): - config = Config( - os.path.join(dir_path, "config.yaml"), num_workers=20, raise_on_error=True - ) - assert config.num_workers == 20 - assert config.raise_on_error - - def test_configuration_normal_override_env(self): - os.environ["NORNIR_NUM_WORKERS"] = "30" - os.environ["NORNIR_RAISE_ON_ERROR"] = "1" - os.environ["NORNIR_SSH_CONFIG_FILE"] = "/user/ssh_config" - config = Config() - assert config.num_workers == 30 - assert config.raise_on_error - assert config.ssh.config_file == "/user/ssh_config" - os.environ.pop("NORNIR_NUM_WORKERS") - os.environ.pop("NORNIR_RAISE_ON_ERROR") - os.environ.pop("NORNIR_SSH_CONFIG_FILE") - - def test_configuration_bool_env(self): - os.environ["NORNIR_RAISE_ON_ERROR"] = "0" - config = Config() - assert config.num_workers == 20 - assert not config.raise_on_error - - def test_get_user_defined_from_file(self): - config = Config(os.path.join(dir_path, "config.yaml")) - assert config.user_defined["asd"] == "qwe" From 46bef2e16a11aa1b7af04ba8e0c407073f67612b Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sat, 13 Oct 2018 16:11:47 +0200 Subject: [PATCH 072/109] refactor config to have everything inside a section --- nornir/core/__init__.py | 6 +- nornir/core/configuration.py | 37 +++--- nornir/core/deserializer/configuration.py | 125 +++++++++++------- nornir/plugins/tasks/text/template_file.py | 4 +- nornir/plugins/tasks/text/template_string.py | 4 +- tests/core/deserializer/test_configuration.py | 63 ++++----- .../test_configuration/config.yaml | 5 +- tests/core/test_InitNornir.py | 13 +- tests/core/test_InitNornir/a_config.yaml | 3 +- 9 files changed, 151 insertions(+), 109 deletions(-) diff --git a/nornir/core/__init__.py b/nornir/core/__init__.py index 68671c15..6a941fb4 100644 --- a/nornir/core/__init__.py +++ b/nornir/core/__init__.py @@ -109,12 +109,12 @@ def run( Raises: :obj:`nornir.core.exceptions.NornirExecutionError`: if at least a task fails - and self.config.raise_on_error is set to ``True`` + and self.config.core.raise_on_error is set to ``True`` Returns: :obj:`nornir.core.task.AggregatedResult`: results of each execution """ - num_workers = num_workers or self.config.num_workers + num_workers = num_workers or self.config.core.num_workers run_on = [] if on_good: @@ -139,7 +139,7 @@ def run( result = self._run_parallel(task, run_on, num_workers, **kwargs) raise_on_error = ( - raise_on_error if raise_on_error is not None else self.config.raise_on_error + raise_on_error if raise_on_error is not None else self.config.core.raise_on_error ) # noqa if raise_on_error: result.raise_on_error() diff --git a/nornir/core/configuration.py b/nornir/core/configuration.py index 5e53e5a0..361675c9 100644 --- a/nornir/core/configuration.py +++ b/nornir/core/configuration.py @@ -1,6 +1,6 @@ import logging import logging.config -from typing import Any, Callable, Dict, List, Optional, Type, TYPE_CHECKING +from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING, Type if TYPE_CHECKING: @@ -87,31 +87,36 @@ def configure(self): logging.config.dictConfig(dictConfig) +class Jinja2Config(object): + __slots__ = "filters" + + def __init__(self, filters: Optional[Dict[str, Callable[..., Any]]]) -> None: + self.filters = filters or {} + + +class CoreConfig(object): + __slots__ = ("num_workers", "raise_on_error") + + def __init__(self, num_workers: int, raise_on_error: bool) -> None: + self.num_workers = num_workers + self.raise_on_error = raise_on_error + + class Config(object): - __slots__ = ( - "inventory", - "jinja_filters", - "num_workers", - "raise_on_error", - "ssh", - "user_defined", - "logging", - ) + __slots__ = ("core", "ssh", "inventory", "jinja2", "logging", "user_defined") def __init__( self, inventory: InventoryConfig, ssh: SSHConfig, logging: LoggingConfig, - jinja_filters: Optional[Dict[str, Callable[..., Any]]], - num_workers: int, - raise_on_error: bool, + jinja2: Jinja2Config, + core: CoreConfig, user_defined: Dict[str, Any], ) -> None: self.inventory = inventory self.ssh = ssh self.logging = logging - self.jinja_filters = jinja_filters or {} - self.num_workers = num_workers - self.raise_on_error = raise_on_error + self.jinja2 = jinja2 + self.core = core self.user_defined = user_defined diff --git a/nornir/core/deserializer/configuration.py b/nornir/core/deserializer/configuration.py index 03bf3248..fef9a2ce 100644 --- a/nornir/core/deserializer/configuration.py +++ b/nornir/core/deserializer/configuration.py @@ -1,11 +1,10 @@ -from typing import Any, Callable, Dict, List, Optional - import importlib import logging +from typing import Any, Callable, Dict, List, Optional from nornir.core import configuration -from pydantic import BaseSettings +from pydantic import BaseSettings, Schema import ruamel.yaml @@ -14,12 +13,9 @@ class SSHConfig(BaseSettings): - """ - Args: - config_file: User ssh_config_file - """ - - config_file: str = "~/.ssh/config" + config_file: str = Schema( + default="~/.ssh/config", description="Path to ssh configuration file" + ) class Config: env_prefix = "NORNIR_SSH_" @@ -28,21 +24,24 @@ class Config: @classmethod def deserialize(self, **kwargs) -> configuration.SSHConfig: s = SSHConfig(**kwargs) - return configuration.SSHConfig(config_file=s.config_file) + return configuration.SSHConfig(**s.dict()) class InventoryConfig(BaseSettings): - """ - Args: - plugin: Path to inventory modules. - transform_function: Path to transform function. The transform_function you provide - will run against each host in the inventory - options: Arguments to pass to the inventory plugin - """ - - plugin: Any = "nornir.plugins.inventory.simple.SimpleInventory" - options: Dict[str, Any] = {} - transform_function: Any = "" + plugin: Any = Schema( + default="nornir.plugins.inventory.simple.SimpleInventory", + description="Import path to inventory plugin", + ) + options: Dict[str, Any] = Schema( + default={}, description="kwargs to pass to the inventory plugin" + ) + transform_function: Any = Schema( + default="", + description=( + "Path to transform function. The transform_function " + "you provide will run against each host in the inventory" + ), + ) class Config: env_prefix = "NORNIR_INVENTORY_" @@ -59,11 +58,16 @@ def deserialize(self, **kwargs) -> configuration.InventoryConfig: class LoggingConfig(BaseSettings): - level: str = "debug" - file: str = "nornir.log" - format: str = "%(asctime)s - %(name)12s - %(levelname)8s - %(funcName)10s() - %(message)s" - to_console: bool = False - loggers: List[str] = ["nornir"] + level: str = Schema(default="debug", description="Logging level") + file: str = Schema(default="nornir.log", descritpion="Logging file") + format: str = Schema( + default="%(asctime)s - %(name)12s - %(levelname)8s - %(funcName)10s() - %(message)s", + description="Logging format", + ) + to_console: bool = Schema( + default=False, description="Whether to log to console or not" + ) + loggers: List[str] = Schema(default=["nornir"], description="Loggers to configure") class Config: env_prefix = "NORNIR_LOGGING_" @@ -81,24 +85,55 @@ def deserialize(self, **kwargs) -> configuration.LoggingConfig: ) -class Config(BaseSettings): - """ - Args: - inventory: Dictionary with Inventory options - jinja_filters: Path to callable returning jinja filters to be used - raise_on_error: If set to ``True``, (:obj:`nornir.core.Nornir.run`) method of - will raise an exception if at least a host failed - num_workers: Number of Nornir worker processes that are run at the same time - configuration can be overridden on individual tasks by using the - """ +class Jinja2Config(BaseSettings): + filters: str = Schema( + default="", description="Path to callable returning jinja filters to be used" + ) + + class Config: + env_prefix = "NORNIR_JINJA2_" + ignore_extra = False + + @classmethod + def deserialize(self, **kwargs) -> configuration.Jinja2Config: + c = Jinja2Config(**kwargs) + jinja_filter_func = _resolve_import_from_string(c.filters) + jinja_filters = jinja_filter_func() if jinja_filter_func else {} + return configuration.Jinja2Config(filters=jinja_filters) + + +class CoreConfig(BaseSettings): + num_workers: int = Schema( + default=20, + description="Number of Nornir worker threads that are run at the same time by default", + ) + raise_on_error: bool = Schema( + default=False, + description=( + "If set to ``True``, (:obj:`nornir.core.Nornir.run`) method of " + "will raise an exception if at least a host failed" + ), + ) + + class Config: + env_prefix = "NORNIR_CORE_" + ignore_extra = False + + @classmethod + def deserialize(self, **kwargs) -> configuration.CoreConfig: + c = CoreConfig(**kwargs) + return configuration.CoreConfig(**c.dict()) + +class Config(BaseSettings): + core: CoreConfig = CoreConfig() inventory: InventoryConfig = InventoryConfig() ssh: SSHConfig = SSHConfig() logging: LoggingConfig = LoggingConfig() - jinja_filters: str = "" - num_workers: int = 20 - raise_on_error: bool = False - user_defined: Dict[str, Any] = {} + jinja2: Jinja2Config = Jinja2Config() + user_defined: Dict[str, Any] = Schema( + default={}, description="User-defined pairs" + ) class Config: env_prefix = "NORNIR_" @@ -107,21 +142,19 @@ class Config: @classmethod def deserialize(cls, **kwargs) -> configuration.Config: c = Config( + core=CoreConfig(**kwargs.pop("core", {})), ssh=SSHConfig(**kwargs.pop("ssh", {})), inventory=InventoryConfig(**kwargs.pop("inventory", {})), logging=LoggingConfig(**kwargs.pop("logging", {})), + jinja2=Jinja2Config(**kwargs.pop("jinja2", {})), **kwargs, ) - - jinja_filter_func = _resolve_import_from_string(c.jinja_filters) - jinja_filters = jinja_filter_func() if jinja_filter_func else {} return configuration.Config( + core=CoreConfig.deserialize(**c.core.dict()), inventory=InventoryConfig.deserialize(**c.inventory.dict()), ssh=SSHConfig.deserialize(**c.ssh.dict()), logging=LoggingConfig.deserialize(**c.logging.dict()), - jinja_filters=jinja_filters, - num_workers=c.num_workers, - raise_on_error=c.raise_on_error, + jinja2=Jinja2Config.deserialize(**c.jinja2.dict()), user_defined=c.user_defined, ) diff --git a/nornir/plugins/tasks/text/template_file.py b/nornir/plugins/tasks/text/template_file.py index 4c86962c..38d8d8b7 100644 --- a/nornir/plugins/tasks/text/template_file.py +++ b/nornir/plugins/tasks/text/template_file.py @@ -19,14 +19,14 @@ def template_file( Arguments: template: filename path: path to dir with templates - jinja_filters: jinja filters to enable. Defaults to nornir.config.jinja_filters + jinja_filters: jinja filters to enable. Defaults to nornir.config.jinja2.filters **kwargs: additional data to pass to the template Returns: Result object with the following attributes set: * result (``string``): rendered string """ - jinja_filters = jinja_filters or {} or task.nornir.config.jinja_filters + jinja_filters = jinja_filters or {} or task.nornir.config.jinja2.filters text = jinja_helper.render_from_file( template=template, path=path, diff --git a/nornir/plugins/tasks/text/template_string.py b/nornir/plugins/tasks/text/template_string.py index 1e808c91..103c82c6 100644 --- a/nornir/plugins/tasks/text/template_string.py +++ b/nornir/plugins/tasks/text/template_string.py @@ -14,14 +14,14 @@ def template_string( Arguments: template (string): template string - jinja_filters (dict): jinja filters to enable. Defaults to nornir.config.jinja_filters + jinja_filters (dict): jinja filters to enable. Defaults to nornir.config.jinja2.filters **kwargs: additional data to pass to the template Returns: Result object with the following attributes set: * result (``string``): rendered string """ - jinja_filters = jinja_filters or {} or task.nornir.config.jinja_filters + jinja_filters = jinja_filters or {} or task.nornir.config.jinja2.filters text = jinja_helper.render_from_string( template=template, host=task.host, jinja_filters=jinja_filters, **kwargs ) diff --git a/tests/core/deserializer/test_configuration.py b/tests/core/deserializer/test_configuration.py index c6ce27a9..e08dfecd 100644 --- a/tests/core/deserializer/test_configuration.py +++ b/tests/core/deserializer/test_configuration.py @@ -37,15 +37,16 @@ def test_config_defaults(self): "to_console": False, "loggers": ["nornir"], }, - "jinja_filters": "", - "num_workers": 20, - "raise_on_error": False, + "jinja2": {"filters": ""}, + "core": {"num_workers": 20, "raise_on_error": False}, "user_defined": {}, } def test_config_basic(self): c = ConfigDeserializer( - num_workers=30, logging={"file": ""}, user_defined={"my_opt": True} + core={"num_workers": 30}, + logging={"file": ""}, + user_defined={"my_opt": True}, ) assert c.dict() == { "inventory": { @@ -61,9 +62,8 @@ def test_config_basic(self): "to_console": False, "loggers": ["nornir"], }, - "jinja_filters": "", - "num_workers": 30, - "raise_on_error": False, + "jinja2": {"filters": ""}, + "core": {"num_workers": 30, "raise_on_error": False}, "user_defined": {"my_opt": True}, } @@ -71,8 +71,8 @@ def test_deserialize_defaults(self): c = ConfigDeserializer.deserialize() assert isinstance(c, Config) - assert c.num_workers == 20 - assert not c.raise_on_error + assert c.core.num_workers == 20 + assert not c.core.raise_on_error assert c.user_defined == {} assert c.logging.level == logging.DEBUG @@ -89,7 +89,7 @@ def test_deserialize_defaults(self): def test_deserialize_basic(self): c = ConfigDeserializer.deserialize( - num_workers=30, + core={"num_workers": 30}, user_defined={"my_opt": True}, logging={"file": "", "level": "info"}, ssh={"config_file": "~/.ssh/alt_config"}, @@ -97,8 +97,8 @@ def test_deserialize_basic(self): ) assert isinstance(c, Config) - assert c.num_workers == 30 - assert not c.raise_on_error + assert c.core.num_workers == 30 + assert not c.core.raise_on_error assert c.user_defined == {"my_opt": True} assert c.logging.level == logging.INFO @@ -115,55 +115,56 @@ def test_deserialize_basic(self): def test_jinja_filters(self): c = ConfigDeserializer.deserialize( - jinja_filters="tests.core.deserializer.my_jinja_filters.jinja_filters" + jinja2={"filters": "tests.core.deserializer.my_jinja_filters.jinja_filters"} ) - assert c.jinja_filters == my_jinja_filters.jinja_filters() + assert c.jinja2.filters == my_jinja_filters.jinja_filters() def test_jinja_filters_error(self): with pytest.raises(ModuleNotFoundError): - ConfigDeserializer.deserialize(jinja_filters="asdasd.asdasd") + ConfigDeserializer.deserialize(jinja2={"filters": "asdasd.asdasd"}) def test_configuration_file_empty(self): config = ConfigDeserializer.load_from_file( os.path.join(dir_path, "empty.yaml"), user_defined={"asd": "qwe"} ) assert config.user_defined["asd"] == "qwe" - assert config.num_workers == 20 - assert not config.raise_on_error + assert config.core.num_workers == 20 + assert not config.core.raise_on_error assert config.inventory.plugin == SimpleInventory def test_configuration_file_normal(self): config = ConfigDeserializer.load_from_file( os.path.join(dir_path, "config.yaml") ) - assert config.num_workers == 10 - assert not config.raise_on_error + assert config.core.num_workers == 10 + assert not config.core.raise_on_error assert config.inventory.plugin == AnsibleInventory def test_configuration_file_override_argument(self): config = ConfigDeserializer.load_from_file( - os.path.join(dir_path, "config.yaml"), num_workers=20, raise_on_error=True + os.path.join(dir_path, "config.yaml"), + core={"num_workers": 20, "raise_on_error": True}, ) - assert config.num_workers == 20 - assert config.raise_on_error + assert config.core.num_workers == 20 + assert config.core.raise_on_error def test_configuration_file_override_env(self): - os.environ["NORNIR_NUM_WORKERS"] = "30" - os.environ["NORNIR_RAISE_ON_ERROR"] = "1" + os.environ["NORNIR_CORE_NUM_WORKERS"] = "30" + os.environ["NORNIR_CORE_RAISE_ON_ERROR"] = "1" os.environ["NORNIR_SSH_CONFIG_FILE"] = "/user/ssh_config" config = ConfigDeserializer.deserialize() - assert config.num_workers == 30 - assert config.raise_on_error + assert config.core.num_workers == 30 + assert config.core.raise_on_error assert config.ssh.config_file == "/user/ssh_config" - os.environ.pop("NORNIR_NUM_WORKERS") - os.environ.pop("NORNIR_RAISE_ON_ERROR") + os.environ.pop("NORNIR_CORE_NUM_WORKERS") + os.environ.pop("NORNIR_CORE_RAISE_ON_ERROR") os.environ.pop("NORNIR_SSH_CONFIG_FILE") def test_configuration_bool_env(self): - os.environ["NORNIR_RAISE_ON_ERROR"] = "0" + os.environ["NORNIR_CORE_RAISE_ON_ERROR"] = "0" config = ConfigDeserializer.deserialize() - assert config.num_workers == 20 - assert not config.raise_on_error + assert config.core.num_workers == 20 + assert not config.core.raise_on_error def test_get_user_defined_from_file(self): config = ConfigDeserializer.load_from_file( diff --git a/tests/core/deserializer/test_configuration/config.yaml b/tests/core/deserializer/test_configuration/config.yaml index ebd6f387..c9a37583 100644 --- a/tests/core/deserializer/test_configuration/config.yaml +++ b/tests/core/deserializer/test_configuration/config.yaml @@ -1,6 +1,7 @@ --- -num_workers: 10 -raise_on_error: false +core: + num_workers: 10 + raise_on_error: false inventory: plugin: nornir.plugins.inventory.ansible.AnsibleInventory user_defined: diff --git a/tests/core/test_InitNornir.py b/tests/core/test_InitNornir.py index 42e05c79..85bbf6b4 100644 --- a/tests/core/test_InitNornir.py +++ b/tests/core/test_InitNornir.py @@ -26,20 +26,20 @@ def test_InitNornir_defaults(self): finally: os.chdir("../../") assert not nr.state.dry_run - assert nr.config.num_workers == 20 + assert nr.config.core.num_workers == 20 assert len(nr.inventory.hosts) assert len(nr.inventory.groups) def test_InitNornir_file(self): nr = InitNornir(config_file=os.path.join(dir_path, "a_config.yaml")) assert not nr.state.dry_run - assert nr.config.num_workers == 100 + assert nr.config.core.num_workers == 100 assert len(nr.inventory.hosts) assert len(nr.inventory.groups) def test_InitNornir_programmatically(self): nr = InitNornir( - num_workers=100, + core={"num_workers": 100}, inventory={ "plugin": "nornir.plugins.inventory.simple.SimpleInventory", "options": { @@ -49,16 +49,17 @@ def test_InitNornir_programmatically(self): }, ) assert not nr.state.dry_run - assert nr.config.num_workers == 100 + assert nr.config.core.num_workers == 100 assert len(nr.inventory.hosts) assert len(nr.inventory.groups) def test_InitNornir_combined(self): nr = InitNornir( - config_file=os.path.join(dir_path, "a_config.yaml"), num_workers=200 + config_file=os.path.join(dir_path, "a_config.yaml"), + core={"num_workers": 200}, ) assert not nr.state.dry_run - assert nr.config.num_workers == 200 + assert nr.config.core.num_workers == 200 assert len(nr.inventory.hosts) assert len(nr.inventory.groups) diff --git a/tests/core/test_InitNornir/a_config.yaml b/tests/core/test_InitNornir/a_config.yaml index abc15dd3..ccf8712b 100644 --- a/tests/core/test_InitNornir/a_config.yaml +++ b/tests/core/test_InitNornir/a_config.yaml @@ -1,5 +1,6 @@ --- -num_workers: 100 +core: + num_workers: 100 inventory: plugin: nornir.plugins.inventory.simple.SimpleInventory options: From 067b04230d7cfacfb3635762dc3d332147477ca9 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sat, 13 Oct 2018 16:20:04 +0200 Subject: [PATCH 073/109] document the configuration --- .../configuration-parameters.j2 | 44 +++++++++---------- docs/conf.py | 21 ++++++++- docs/configuration/index.rst | 23 +++++++++- docs/howto/advanced_filtering/config.yaml | 2 + .../tutorials/intro/initializing_nornir.ipynb | 2 +- docs/upgrading/1_to_2.rst | 5 +++ 6 files changed, 71 insertions(+), 26 deletions(-) diff --git a/docs/_data_templates/configuration-parameters.j2 b/docs/_data_templates/configuration-parameters.j2 index cecf3313..5543753a 100644 --- a/docs/_data_templates/configuration-parameters.j2 +++ b/docs/_data_templates/configuration-parameters.j2 @@ -1,29 +1,29 @@ -The configuration parameters will be set by the :doc:`Nornir.core.configuration.Config ` class. +{% macro document_section(section, schema) %} +{{ section }} +{{ "-" * section|length }} -{% for k, v in params|dictsort %} ----------- +{% for k, v in schema["properties"].items() -%} -{{ k }} ----------------------------------- +``{{ k }}`` +{{ "_" * (k|length + 4) }} +.. list-table:: + :widths: 15 85 -.. raw:: html + * - **Descrpiption** + - {{ v["description"] }} + * - **Type** + - ``{{ v["type"] }}`` + * - **Default** + - {{ "``{}``".format(v["default"]) if v["default"] else "" }} + * - **Required** + - ``{{ v["required"] }}`` + * - **Environment Variable** + - ``{{ "NORNIR_{}_{}".format(section, k).upper() }}`` - - - - - - {% if v['type'] in ('str', 'int', 'bool') %} - - {% else %} - - {% endif %} - - - -
Environment variableTypeDefault
{{ v['env'] or 'BRIGADE_' + k|upper }}N/A{{ v['type'] }}{{ v['default_doc'] or v['default'] }}
- -{{ v['description'] }} +{% endfor %} +{%- endmacro %} +{% for k, v in schema["properties"].items() if k not in ["user_defined"] %} +{{ document_section(k, v) }} {% endfor %} diff --git a/docs/conf.py b/docs/conf.py index eab549db..4a52e526 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,11 +20,15 @@ import os import sys +from jinja2 import Environment, FileSystemLoader + sys.path.insert(0, os.path.abspath("../")) +from nornir.core.deserializer.configuration import Config + # -- General configuration ------------------------------------------------ -BASEPATH = os.path.dirname(__file__) +BASEPATH = os.path.abspath(os.path.dirname(__file__)) # If your documentation needs a minimal Sphinx version, state it here. # @@ -49,7 +53,7 @@ # General information about the project. project = "nornir" -copyright = "2017, David Barroso" +copyright = "2018, David Barroso" author = "David Barroso" # The version info for the project you're documenting, acts as replacement for @@ -174,7 +178,20 @@ def skip_slots(app, what, name, obj, skip, options): return None +def build_configuration_parameters(app): + """Create documentation for configuration parameters.""" + env = Environment(loader=FileSystemLoader("{0}/_data_templates".format(BASEPATH))) + template_file = env.get_template("configuration-parameters.j2") + data = {} + data["schema"] = Config.schema() + rendered_template = template_file.render(**data) + output_dir = "{0}/configuration/generated".format(BASEPATH) + with open("{}/parameters.rst".format(output_dir), "w") as f: + f.write(rendered_template) + + def setup(app): """Map methods to states of the documentation build.""" + app.connect("builder-inited", build_configuration_parameters) app.connect("autodoc-skip-member", skip_slots) app.add_stylesheet("css/custom.css") diff --git a/docs/configuration/index.rst b/docs/configuration/index.rst index 6d292c38..83c56ce6 100644 --- a/docs/configuration/index.rst +++ b/docs/configuration/index.rst @@ -1,4 +1,25 @@ Configuration ============= -# TODO +The configuration is comprised of a set of sections and parameters for those sections. You can set the configuration programmatically using nornir by passing a dictionary of options for each section, by using a YAML file, by setting the corresponding environment variables or by a combination of the three. The order of preference from less to more preferred is "configuration file" -> "env variable" -> "code". + +An example using ``InitNornir`` would be:: + + nr = InitNornir( + core={"num_workers": 20}, + logging={"file": "mylogs", "level": "debug"} + ) + +A similar example using a ``yaml`` file: + +.. include:: ../howto/advanced_filtering/config.yaml + :code: yaml + +A continuation you can find each section and their corresponding options. + +.. include:: generated/parameters.rst + +user_defined +------------ + +You can set any ```` pair you want here and you will have it available under your configuration object, i.e. ``nr.config.user_defined.my_app_option``. diff --git a/docs/howto/advanced_filtering/config.yaml b/docs/howto/advanced_filtering/config.yaml index 25cd0ad1..a084ae01 100644 --- a/docs/howto/advanced_filtering/config.yaml +++ b/docs/howto/advanced_filtering/config.yaml @@ -1,4 +1,6 @@ --- +core: + num_workers: 20 inventory: plugin: nornir.plugins.inventory.simple.SimpleInventory options: diff --git a/docs/tutorials/intro/initializing_nornir.ipynb b/docs/tutorials/intro/initializing_nornir.ipynb index f7ff577a..7270055c 100644 --- a/docs/tutorials/intro/initializing_nornir.ipynb +++ b/docs/tutorials/intro/initializing_nornir.ipynb @@ -204,7 +204,7 @@ } ], "source": [ - "nr.config.num_workers" + "nr.config.core.num_workers" ] } ], diff --git a/docs/upgrading/1_to_2.rst b/docs/upgrading/1_to_2.rst index 1ce7e92f..d11fcfe8 100644 --- a/docs/upgrading/1_to_2.rst +++ b/docs/upgrading/1_to_2.rst @@ -24,3 +24,8 @@ In order to import ``InitNornir`` correctly you have to change the old path:: to:: from nornir import InitNornir + +Changes to the configuration +---------------------------- + +The format of the configuration has slightly changed. Some of the options that used to be under the root object, for instance ``num_workers``, ``jinja_filters`` and ``raise_on_error`` are now under ``core`` and ``jinja2`` sections. For details, go to the `configuration section <../configuration/index.rst>`_ From 32c69ebac55477e2523aae148334861269cd40d1 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sat, 13 Oct 2018 16:31:07 +0200 Subject: [PATCH 074/109] fix docs for exceptions --- docs/ref/api/exceptions.rst | 21 +-------------------- nornir/core/exceptions.py | 26 +++++++++++++++++++++++--- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/docs/ref/api/exceptions.rst b/docs/ref/api/exceptions.rst index 74b37c5a..282225e3 100644 --- a/docs/ref/api/exceptions.rst +++ b/docs/ref/api/exceptions.rst @@ -1,25 +1,6 @@ Exceptions ========== -CommandError ------------- - -.. autoclass:: nornir.core.exceptions.CommandError - :members: - :undoc-members: - - -NornirExecutionError --------------------- - -.. autoclass:: nornir.core.exceptions.NornirExecutionError +.. automodule:: nornir.core.exceptions :members: :undoc-members: - -NornirSubTaskError ------------------- - -.. autoclass:: nornir.core.exceptions.NornirSubTaskError - :members: - :undoc-members: - diff --git a/nornir/core/exceptions.py b/nornir/core/exceptions.py index e1683be9..94a72b00 100644 --- a/nornir/core/exceptions.py +++ b/nornir/core/exceptions.py @@ -1,24 +1,41 @@ -from builtins import super - - class ConnectionException(Exception): + """ + Superclass for all the Connection* Exceptions + """ + def __init__(self, connection): self.connection = connection class ConnectionAlreadyOpen(ConnectionException): + """ + Raised when opening an already opened connection + """ + pass class ConnectionNotOpen(ConnectionException): + """ + Raised when trying to close a connection that isn't open + """ + pass class ConnectionPluginAlreadyRegistered(ConnectionException): + """ + Raised when trying to register an already registered plugin + """ + pass class ConnectionPluginNotRegistered(ConnectionException): + """ + Raised when trying to access a plugin that is not registered + """ + pass @@ -57,6 +74,9 @@ def __init__(self, result): @property def failed_hosts(self): + """ + Hosts that failed to complete the task + """ return {k: v for k, v in self.result.items() if v.failed} def __str__(self): From a639ce0fe8f86f8bdb209533656922f13bc8ac7d Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sat, 13 Oct 2018 16:32:23 +0200 Subject: [PATCH 075/109] blackify --- nornir/core/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nornir/core/__init__.py b/nornir/core/__init__.py index 6a941fb4..ec94842c 100644 --- a/nornir/core/__init__.py +++ b/nornir/core/__init__.py @@ -139,7 +139,9 @@ def run( result = self._run_parallel(task, run_on, num_workers, **kwargs) raise_on_error = ( - raise_on_error if raise_on_error is not None else self.config.core.raise_on_error + raise_on_error + if raise_on_error is not None + else self.config.core.raise_on_error ) # noqa if raise_on_error: result.raise_on_error() From 60f336c6405fd089274a03161322bc74928b856e Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sat, 13 Oct 2018 16:33:21 +0200 Subject: [PATCH 076/109] pylamafy --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 4a52e526..8f2c0502 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,7 +24,7 @@ sys.path.insert(0, os.path.abspath("../")) -from nornir.core.deserializer.configuration import Config +from nornir.core.deserializer.configuration import Config # noqa # -- General configuration ------------------------------------------------ From 26861733523f4a81945f9323a2f68b5ba155d949 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sat, 13 Oct 2018 16:45:51 +0200 Subject: [PATCH 077/109] fix mypy --- nornir/core/deserializer/configuration.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nornir/core/deserializer/configuration.py b/nornir/core/deserializer/configuration.py index fef9a2ce..c2f37f0f 100644 --- a/nornir/core/deserializer/configuration.py +++ b/nornir/core/deserializer/configuration.py @@ -1,8 +1,9 @@ import importlib import logging -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Type, cast from nornir.core import configuration +from nornir.core.inventory import Inventory from pydantic import BaseSettings, Schema @@ -51,7 +52,7 @@ class Config: def deserialize(self, **kwargs) -> configuration.InventoryConfig: inv = InventoryConfig(**kwargs) return configuration.InventoryConfig( - plugin=_resolve_import_from_string(inv.plugin), + plugin=cast(Type[Inventory], _resolve_import_from_string(inv.plugin)), options=inv.options, transform_function=_resolve_import_from_string(inv.transform_function), ) @@ -160,7 +161,7 @@ def deserialize(cls, **kwargs) -> configuration.Config: @classmethod def load_from_file(cls, config_file: str, **kwargs) -> configuration.Config: - config_dict = {} + config_dict: Dict[str, Any] = {} if config_file: yml = ruamel.yaml.YAML(typ="safe") with open(config_file, "r") as f: From b34d09c137f4c99476b2dfc711878a1ceaea82eb Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 14 Oct 2018 13:01:29 +0200 Subject: [PATCH 078/109] add missing directory --- docs/configuration/generated/.placeholder | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/configuration/generated/.placeholder diff --git a/docs/configuration/generated/.placeholder b/docs/configuration/generated/.placeholder new file mode 100644 index 00000000..e69de29b From 29a7a9edd365aa4457e7cf0359b8f2b7c5137437 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Tue, 16 Oct 2018 09:55:03 +0200 Subject: [PATCH 079/109] fix typos --- docs/_data_templates/configuration-parameters.j2 | 2 +- docs/configuration/index.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/_data_templates/configuration-parameters.j2 b/docs/_data_templates/configuration-parameters.j2 index 5543753a..e031415f 100644 --- a/docs/_data_templates/configuration-parameters.j2 +++ b/docs/_data_templates/configuration-parameters.j2 @@ -10,7 +10,7 @@ .. list-table:: :widths: 15 85 - * - **Descrpiption** + * - **Description** - {{ v["description"] }} * - **Type** - ``{{ v["type"] }}`` diff --git a/docs/configuration/index.rst b/docs/configuration/index.rst index 83c56ce6..df1cffff 100644 --- a/docs/configuration/index.rst +++ b/docs/configuration/index.rst @@ -15,7 +15,7 @@ A similar example using a ``yaml`` file: .. include:: ../howto/advanced_filtering/config.yaml :code: yaml -A continuation you can find each section and their corresponding options. +Next, you can find each section and their corresponding options. .. include:: generated/parameters.rst From 78fcec9eaaa5ec2b6bff73ce914a02b78779f029 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Tue, 16 Oct 2018 10:01:09 +0200 Subject: [PATCH 080/109] fixes #258 --- nornir/plugins/inventory/simple.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nornir/plugins/inventory/simple.py b/nornir/plugins/inventory/simple.py index 25d5bb35..fc434de6 100644 --- a/nornir/plugins/inventory/simple.py +++ b/nornir/plugins/inventory/simple.py @@ -1,7 +1,7 @@ import logging import os -from nornir.core.deserializer.inventory import Inventory, VarsDict, GroupsDict +from nornir.core.deserializer.inventory import GroupsDict, Inventory, VarsDict import ruamel.yaml @@ -23,17 +23,17 @@ def __init__( if group_file: if os.path.exists(group_file): with open(group_file, "r") as f: - groups = yml.load(f) + groups = yml.load(f) or {} else: - logging.warning("{}: doesn't exist".format(group_file)) + logging.debug("{}: doesn't exist".format(group_file)) groups = {} defaults: VarsDict = {} if defaults_file: if os.path.exists(defaults_file): with open(defaults_file, "r") as f: - defaults = yml.load(f) + defaults = yml.load(f) or {} else: - logging.warning("{}: doesn't exist".format(defaults_file)) + logging.debug("{}: doesn't exist".format(defaults_file)) defaults = {} super().__init__(hosts=hosts, groups=groups, defaults=defaults, *args, **kwargs) From 7c7c85cbcf59007d1f00122076933311073de482 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Fri, 19 Oct 2018 10:01:35 +0200 Subject: [PATCH 081/109] change logger names __name__ --- nornir/core/__init__.py | 2 +- nornir/core/deserializer/configuration.py | 2 +- nornir/core/task.py | 2 +- nornir/plugins/inventory/ansible.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nornir/core/__init__.py b/nornir/core/__init__.py index ec94842c..34fbfba2 100644 --- a/nornir/core/__init__.py +++ b/nornir/core/__init__.py @@ -30,7 +30,7 @@ def __init__( self, inventory, _config=None, config_file=None, logger=None, data=None ): self.data = data if data is not None else GlobalState() - self.logger = logger or logging.getLogger("nornir") + self.logger = logger or logging.getLogger(__name__) self.inventory = inventory diff --git a/nornir/core/deserializer/configuration.py b/nornir/core/deserializer/configuration.py index c2f37f0f..6b186d29 100644 --- a/nornir/core/deserializer/configuration.py +++ b/nornir/core/deserializer/configuration.py @@ -10,7 +10,7 @@ import ruamel.yaml -logger = logging.getLogger("nornir") +logger = logging.getLogger(__name__) class SSHConfig(BaseSettings): diff --git a/nornir/core/task.py b/nornir/core/task.py index ee190343..b80619bf 100644 --- a/nornir/core/task.py +++ b/nornir/core/task.py @@ -56,7 +56,7 @@ def start(self, host, nornir): self.host = host self.nornir = nornir - logger = logging.getLogger("nornir") + logger = logging.getLogger(__name__) try: logger.info("{}: {}: running task".format(self.host.name, self.name)) r = self.task(self, **self.params) diff --git a/nornir/plugins/inventory/ansible.py b/nornir/plugins/inventory/ansible.py index a5d45226..b0066b03 100644 --- a/nornir/plugins/inventory/ansible.py +++ b/nornir/plugins/inventory/ansible.py @@ -25,7 +25,7 @@ YAML = ruamel.yaml.YAML(typ="safe") -logger = logging.getLogger("nornir") +logger = logging.getLogger(__name__) AnsibleHostsDict = Dict[str, Optional[VarsDict]] From 75820ae8cf9ede8959937c1930ae87fc9b90a991 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Fri, 19 Oct 2018 10:08:52 +0200 Subject: [PATCH 082/109] fix deserializer types --- nornir/core/deserializer/configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nornir/core/deserializer/configuration.py b/nornir/core/deserializer/configuration.py index 6b186d29..4c400e57 100644 --- a/nornir/core/deserializer/configuration.py +++ b/nornir/core/deserializer/configuration.py @@ -29,14 +29,14 @@ def deserialize(self, **kwargs) -> configuration.SSHConfig: class InventoryConfig(BaseSettings): - plugin: Any = Schema( + plugin: str = Schema( default="nornir.plugins.inventory.simple.SimpleInventory", description="Import path to inventory plugin", ) options: Dict[str, Any] = Schema( default={}, description="kwargs to pass to the inventory plugin" ) - transform_function: Any = Schema( + transform_function: str = Schema( default="", description=( "Path to transform function. The transform_function " From 147ef56e3ae97056457b20461fcd57370bfbd9db Mon Sep 17 00:00:00 2001 From: David Barroso Date: Fri, 19 Oct 2018 10:09:35 +0200 Subject: [PATCH 083/109] classmethod should follow cls convention --- nornir/core/deserializer/configuration.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nornir/core/deserializer/configuration.py b/nornir/core/deserializer/configuration.py index 4c400e57..03b313a1 100644 --- a/nornir/core/deserializer/configuration.py +++ b/nornir/core/deserializer/configuration.py @@ -23,7 +23,7 @@ class Config: ignore_extra = False @classmethod - def deserialize(self, **kwargs) -> configuration.SSHConfig: + def deserialize(cls, **kwargs) -> configuration.SSHConfig: s = SSHConfig(**kwargs) return configuration.SSHConfig(**s.dict()) @@ -49,7 +49,7 @@ class Config: ignore_extra = False @classmethod - def deserialize(self, **kwargs) -> configuration.InventoryConfig: + def deserialize(cls, **kwargs) -> configuration.InventoryConfig: inv = InventoryConfig(**kwargs) return configuration.InventoryConfig( plugin=cast(Type[Inventory], _resolve_import_from_string(inv.plugin)), @@ -75,7 +75,7 @@ class Config: ignore_extra = False @classmethod - def deserialize(self, **kwargs) -> configuration.LoggingConfig: + def deserialize(cls, **kwargs) -> configuration.LoggingConfig: logging_config = LoggingConfig(**kwargs) return configuration.LoggingConfig( level=getattr(logging, logging_config.level.upper()), @@ -96,7 +96,7 @@ class Config: ignore_extra = False @classmethod - def deserialize(self, **kwargs) -> configuration.Jinja2Config: + def deserialize(cls, **kwargs) -> configuration.Jinja2Config: c = Jinja2Config(**kwargs) jinja_filter_func = _resolve_import_from_string(c.filters) jinja_filters = jinja_filter_func() if jinja_filter_func else {} @@ -121,7 +121,7 @@ class Config: ignore_extra = False @classmethod - def deserialize(self, **kwargs) -> configuration.CoreConfig: + def deserialize(cls, **kwargs) -> configuration.CoreConfig: c = CoreConfig(**kwargs) return configuration.CoreConfig(**c.dict()) From 19246499a61414472d2868cc7dba8b763c42b04a Mon Sep 17 00:00:00 2001 From: David Barroso Date: Fri, 19 Oct 2018 10:44:29 +0200 Subject: [PATCH 084/109] various fixes --- nornir/core/deserializer/configuration.py | 6 +++--- nornir/init_nornir.py | 19 ++++++++++++++++--- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/nornir/core/deserializer/configuration.py b/nornir/core/deserializer/configuration.py index 03b313a1..cc9984ab 100644 --- a/nornir/core/deserializer/configuration.py +++ b/nornir/core/deserializer/configuration.py @@ -112,7 +112,8 @@ class CoreConfig(BaseSettings): default=False, description=( "If set to ``True``, (:obj:`nornir.core.Nornir.run`) method of " - "will raise an exception if at least a host failed" + "will raise exception :obj:`nornir.core.exceptions.NornirExecutionError` " + "if at least a host failed" ), ) @@ -175,8 +176,7 @@ def _resolve_import_from_string(import_path: Any) -> Optional[Callable[..., Any] return None elif callable(import_path): return import_path - module_name = ".".join(import_path.split(".")[:-1]) - obj_name = import_path.split(".")[-1] + module_name, obj_name = import_path.rsplit(".", 1) module = importlib.import_module(module_name) return getattr(module, obj_name) except Exception as e: diff --git a/nornir/init_nornir.py b/nornir/init_nornir.py index 1f3beea7..9b12a037 100644 --- a/nornir/init_nornir.py +++ b/nornir/init_nornir.py @@ -1,8 +1,9 @@ +from typing import Any, Callable + from nornir.core import Nornir +from nornir.core.connections import Connections from nornir.core.deserializer.configuration import Config from nornir.core.state import GlobalState -from nornir.core.connections import Connections - from nornir.plugins.connections.napalm import Napalm from nornir.plugins.connections.netmiko import Netmiko from nornir.plugins.connections.paramiko import Paramiko @@ -14,6 +15,10 @@ def register_default_connection_plugins() -> None: Connections.register("paramiko", Paramiko) +def cls_to_string(cls: Callable[..., Any]) -> str: + return f"{cls.__module__}.{cls.__name__}" + + def InitNornir(config_file="", dry_run=False, configure_logging=True, **kwargs): """ Arguments: @@ -27,6 +32,14 @@ def InitNornir(config_file="", dry_run=False, configure_logging=True, **kwargs): """ register_default_connection_plugins() + if callable(kwargs.get("inventory", {}).get("plugin", "")): + kwargs["inventory"]["plugin"] = cls_to_string(kwargs["inventory"]["plugin"]) + + if callable(kwargs.get("inventory", {}).get("transform_function", "")): + kwargs["inventory"]["transform_function"] = cls_to_string( + kwargs["inventory"]["transform_function"] + ) + conf = Config.load_from_file(config_file, **kwargs) GlobalState.dry_run = dry_run @@ -37,7 +50,7 @@ def InitNornir(config_file="", dry_run=False, configure_logging=True, **kwargs): inv = conf.inventory.plugin.deserialize( transform_function=conf.inventory.transform_function, config=conf, - **conf.inventory.options + **conf.inventory.options, ) return Nornir(inventory=inv, _config=conf) From 37a72b1782f6d43e0e9b93a995bd7111258a51d4 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Tue, 23 Oct 2018 18:13:31 +0200 Subject: [PATCH 085/109] addressing PR comments --- docs/howto/inventory.ipynb | 2 +- nornir/core/state.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/howto/inventory.ipynb b/docs/howto/inventory.ipynb index 3dc7f693..3ca5b21c 100644 --- a/docs/howto/inventory.ipynb +++ b/docs/howto/inventory.ipynb @@ -1366,7 +1366,7 @@ "source": [ "##### Filter Object\n", "\n", - "You can also use a filter object to create incrementally a complext query object. Let's see how it works by example:" + "You can also use a filter object to create incrementally a complex query object. Let's see how it works by example:" ] }, { diff --git a/nornir/core/state.py b/nornir/core/state.py index b10a6f4c..311b2633 100644 --- a/nornir/core/state.py +++ b/nornir/core/state.py @@ -4,7 +4,7 @@ class GlobalState(object): versions of Nornir after running ``filter`` multiple times. Attributes: - failed_hosts (list): Hosts that have failed to run a task properly + failed_hosts: Hosts that have failed to run a task properly """ def __init__(self): From 0d358887f4d46468a84fc148e664da596d1d90da Mon Sep 17 00:00:00 2001 From: David Barroso Date: Wed, 24 Oct 2018 14:42:12 +0200 Subject: [PATCH 086/109] remove need of Config object in the Inventory and Hosts --- nornir/core/__init__.py | 19 +------- nornir/core/configuration.py | 2 +- nornir/core/deserializer/configuration.py | 2 +- nornir/core/deserializer/inventory.py | 4 +- nornir/core/inventory.py | 46 ++++--------------- nornir/init_nornir.py | 2 +- .../plugins/tasks/commands/remote_command.py | 2 +- nornir/plugins/tasks/files/sftp.py | 2 +- nornir/plugins/tasks/networking/napalm_cli.py | 2 +- .../tasks/networking/napalm_configure.py | 2 +- nornir/plugins/tasks/networking/napalm_get.py | 2 +- .../tasks/networking/napalm_validate.py | 2 +- .../tasks/networking/netmiko_file_transfer.py | 2 +- .../tasks/networking/netmiko_send_command.py | 2 +- .../tasks/networking/netmiko_send_config.py | 2 +- tests/core/test_connections.py | 10 ++-- .../functions/text/test_print_result.py | 5 +- .../tasks/networking/test_napalm_cli.py | 5 +- .../tasks/networking/test_napalm_configure.py | 5 +- .../tasks/networking/test_napalm_get.py | 5 +- .../tasks/networking/test_napalm_validate.py | 5 +- 21 files changed, 49 insertions(+), 79 deletions(-) diff --git a/nornir/core/__init__.py b/nornir/core/__init__.py index 34fbfba2..8b8a4ccd 100644 --- a/nornir/core/__init__.py +++ b/nornir/core/__init__.py @@ -17,7 +17,6 @@ class Nornir(object): data(GlobalState): shared data amongst different iterations of nornir dry_run(``bool``): Whether if we are testing the changes or not config (:obj:`nornir.core.configuration.Config`): Configuration object - config_file (``str``): Path to Yaml configuration file Attributes: inventory (:obj:`nornir.core.inventory.Inventory`): Inventory to work with @@ -26,18 +25,13 @@ class Nornir(object): config (:obj:`nornir.core.configuration.Config`): Configuration parameters """ - def __init__( - self, inventory, _config=None, config_file=None, logger=None, data=None - ): + def __init__(self, inventory, config=None, logger=None, data=None): self.data = data if data is not None else GlobalState() self.logger = logger or logging.getLogger(__name__) self.inventory = inventory - if config_file: - self._config = Config(config_file=config_file) - else: - self._config = _config or Config() + self.config = config or Config() def __enter__(self): return self @@ -45,15 +39,6 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self.close_connections(on_good=True, on_failed=True) - @property - def config(self): - return self._config - - @config.setter - def config(self, value): - self._config = value - self.inventory.config = value - def filter(self, *args, **kwargs): """ See :py:meth:`nornir.core.inventory.Inventory.filter` diff --git a/nornir/core/configuration.py b/nornir/core/configuration.py index 361675c9..d863a485 100644 --- a/nornir/core/configuration.py +++ b/nornir/core/configuration.py @@ -4,7 +4,7 @@ if TYPE_CHECKING: - from nornir.core.inventory import Inventory # noqa + from nornir.core.deserializer.inventory import Inventory # noqa class SSHConfig(object): diff --git a/nornir/core/deserializer/configuration.py b/nornir/core/deserializer/configuration.py index cc9984ab..39b998db 100644 --- a/nornir/core/deserializer/configuration.py +++ b/nornir/core/deserializer/configuration.py @@ -3,7 +3,7 @@ from typing import Any, Callable, Dict, List, Optional, Type, cast from nornir.core import configuration -from nornir.core.inventory import Inventory +from nornir.core.deserializer.inventory import Inventory from pydantic import BaseSettings, Schema diff --git a/nornir/core/deserializer/inventory.py b/nornir/core/deserializer/inventory.py index 7a4f8e56..a9ebccad 100644 --- a/nornir/core/deserializer/inventory.py +++ b/nornir/core/deserializer/inventory.py @@ -1,7 +1,6 @@ from typing import Any, Dict, List, Optional, Union from nornir.core import inventory -from nornir.core.deserializer.configuration import Config from pydantic import BaseModel @@ -108,7 +107,7 @@ class Inventory(BaseModel): defaults: Defaults @classmethod - def deserialize(cls, config=None, transform_function=None, *args, **kwargs): + def deserialize(cls, transform_function=None, *args, **kwargs): deserialized = cls(*args, **kwargs) defaults_dict = deserialized.defaults.dict() @@ -131,7 +130,6 @@ def deserialize(cls, config=None, transform_function=None, *args, **kwargs): groups=groups, defaults=defaults, transform_function=transform_function, - config=config or Config.deserialize(), ) @classmethod diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 4881f47d..cb857c6e 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -2,7 +2,7 @@ from typing import Any, Dict, List, Optional from nornir.core.configuration import Config -from nornir.core.connections import Connections, ConnectionPlugin +from nornir.core.connections import ConnectionPlugin, Connections from nornir.core.exceptions import ConnectionAlreadyOpen, ConnectionNotOpen @@ -60,20 +60,18 @@ def __contains__(self, value) -> bool: class InventoryElement(BaseAttributes): - __slots__ = ("groups", "data", "connection_options", "config") + __slots__ = ("groups", "data", "connection_options") def __init__( self, groups: Optional[ParentGroups] = None, data: Optional[Dict[str, Any]] = None, connection_options: Optional[Dict[str, ConnectionOptions]] = None, - config: Optional[Config] = None, **kwargs, ) -> None: self.groups = groups or ParentGroups() self.data = data or {} self.connection_options = connection_options or {} - self.config = config super().__init__(**kwargs) @@ -92,14 +90,10 @@ def __init__( class Host(InventoryElement): - __slots__ = ("name", "connections", "defaults", "config") + __slots__ = ("name", "connections", "defaults") def __init__( - self, - name: str, - defaults: Optional[Defaults] = None, - config: Optional[Config] = None, - **kwargs, + self, name: str, defaults: Optional[Defaults] = None, **kwargs ) -> None: self.name = name self.defaults = defaults or Defaults() @@ -281,7 +275,7 @@ def _get_connection_options_recursively( p.extras = p.extras if p.extras is not None else sp.extras return p - def get_connection(self, connection: str) -> Any: + def get_connection(self, connection: str, configuration: Config) -> Any: """ The function of this method is twofold: @@ -301,7 +295,7 @@ def get_connection(self, connection: str) -> Any: if connection not in self.connections: self.open_connection( connection=connection, - configuration=self.config, + configuration=configuration, **self.get_connection_parameters(connection).dict(), ) return self.connections[connection].connection @@ -318,13 +312,13 @@ def get_connection_state(self, connection: str) -> Dict[str, Any]: def open_connection( self, connection: str, + configuration: Config, hostname: Optional[str] = None, username: Optional[str] = None, password: Optional[str] = None, port: Optional[int] = None, platform: Optional[str] = None, extras: Optional[Dict[str, Any]] = None, - configuration: Optional[Config] = None, default_to_host_attributes: bool = True, ) -> ConnectionPlugin: """ @@ -352,9 +346,7 @@ def open_connection( port=port if port is not None else conn_params.port, platform=platform if platform is not None else conn_params.platform, extras=extras if extras is not None else conn_params.extras, - configuration=configuration - if configuration is not None - else self.config, + configuration=configuration, ) else: self.connections[connection].open( @@ -395,14 +387,13 @@ class Groups(Dict[str, Group]): class Inventory(object): - __slots__ = ("hosts", "groups", "defaults", "_config") + __slots__ = ("hosts", "groups", "defaults") def __init__( self, hosts: Hosts, groups: Optional[Groups] = None, defaults: Optional[Defaults] = None, - config: Optional[Config] = None, transform_function=None, ) -> None: self.hosts = hosts @@ -418,8 +409,6 @@ def __init__( for h in self.hosts.values(): transform_function(h) - self.config = config - def filter(self, filter_obj=None, filter_func=None, *args, **kwargs): filter_func = filter_obj or filter_func if filter_func: @@ -430,22 +419,7 @@ def filter(self, filter_obj=None, filter_func=None, *args, **kwargs): for n, h in self.hosts.items() if all(h.get(k) == v for k, v in kwargs.items()) } - return Inventory( - hosts=filtered, - groups=self.groups, - defaults=self.defaults, - config=self.config, - ) + return Inventory(hosts=filtered, groups=self.groups, defaults=self.defaults) def __len__(self): return self.hosts.__len__() - - @property - def config(self): - return self._config - - @config.setter - def config(self, value): - self._config = value - for host in self.hosts.values(): - host.config = value diff --git a/nornir/init_nornir.py b/nornir/init_nornir.py index 9b12a037..caf71a1c 100644 --- a/nornir/init_nornir.py +++ b/nornir/init_nornir.py @@ -53,4 +53,4 @@ def InitNornir(config_file="", dry_run=False, configure_logging=True, **kwargs): **conf.inventory.options, ) - return Nornir(inventory=inv, _config=conf) + return Nornir(inventory=inv, config=conf) diff --git a/nornir/plugins/tasks/commands/remote_command.py b/nornir/plugins/tasks/commands/remote_command.py index b803cb20..dbfbb4a6 100644 --- a/nornir/plugins/tasks/commands/remote_command.py +++ b/nornir/plugins/tasks/commands/remote_command.py @@ -20,7 +20,7 @@ def remote_command(task: Task, command: str) -> Result: Raises: :obj:`nornir.core.exceptions.CommandError`: when there is a command error """ - client = task.host.get_connection("paramiko") + client = task.host.get_connection("paramiko", task.nornir.config) connection_state = task.host.get_connection_state("paramiko") chan = client.get_transport().open_session() diff --git a/nornir/plugins/tasks/files/sftp.py b/nornir/plugins/tasks/files/sftp.py index 2e85308e..66dc62dd 100644 --- a/nornir/plugins/tasks/files/sftp.py +++ b/nornir/plugins/tasks/files/sftp.py @@ -149,7 +149,7 @@ def sftp( """ dry_run = task.is_dry_run(dry_run) actions = {"put": put, "get": get} - client = task.host.get_connection("paramiko") + client = task.host.get_connection("paramiko", task.nornir.config) scp_client = SCPClient(client.get_transport()) sftp_client = paramiko.SFTPClient.from_transport(client.get_transport()) files_changed = actions[action](task, scp_client, sftp_client, src, dst, dry_run) diff --git a/nornir/plugins/tasks/networking/napalm_cli.py b/nornir/plugins/tasks/networking/napalm_cli.py index b6836ca2..3e9f70ca 100644 --- a/nornir/plugins/tasks/networking/napalm_cli.py +++ b/nornir/plugins/tasks/networking/napalm_cli.py @@ -14,6 +14,6 @@ def napalm_cli(task: Task, commands: List[str]) -> Result: Result object with the following attributes set: * result (``dict``): result of the commands execution """ - device = task.host.get_connection("napalm") + device = task.host.get_connection("napalm", task.nornir.config) result = device.cli(commands) return Result(host=task.host, result=result) diff --git a/nornir/plugins/tasks/networking/napalm_configure.py b/nornir/plugins/tasks/networking/napalm_configure.py index 8d674187..518aba8d 100644 --- a/nornir/plugins/tasks/networking/napalm_configure.py +++ b/nornir/plugins/tasks/networking/napalm_configure.py @@ -24,7 +24,7 @@ def napalm_configure( * changed (``bool``): whether the task is changing the system or not * diff (``string``): change in the system """ - device = task.host.get_connection("napalm") + device = task.host.get_connection("napalm", task.nornir.config) if replace: device.load_replace_candidate(filename=filename, config=configuration) diff --git a/nornir/plugins/tasks/networking/napalm_get.py b/nornir/plugins/tasks/networking/napalm_get.py index 8fee7de8..2c34bf76 100644 --- a/nornir/plugins/tasks/networking/napalm_get.py +++ b/nornir/plugins/tasks/networking/napalm_get.py @@ -46,7 +46,7 @@ def napalm_get( Result object with the following attributes set: * result (``dict``): dictionary with the result of the getter """ - device = task.host.get_connection("napalm") + device = task.host.get_connection("napalm", task.nornir.config) getters_options = getters_options or {} if isinstance(getters, str): diff --git a/nornir/plugins/tasks/networking/napalm_validate.py b/nornir/plugins/tasks/networking/napalm_validate.py index 2a747385..19498e99 100644 --- a/nornir/plugins/tasks/networking/napalm_validate.py +++ b/nornir/plugins/tasks/networking/napalm_validate.py @@ -24,7 +24,7 @@ def napalm_validate( * result (``dict``): dictionary with the result of the validation * complies (``bool``): Whether the device complies or not """ - device = task.host.get_connection("napalm") + device = task.host.get_connection("napalm", task.nornir.config) r = device.compliance_report( validation_file=src, validation_source=validation_source ) diff --git a/nornir/plugins/tasks/networking/netmiko_file_transfer.py b/nornir/plugins/tasks/networking/netmiko_file_transfer.py index b47c7950..96951ab9 100644 --- a/nornir/plugins/tasks/networking/netmiko_file_transfer.py +++ b/nornir/plugins/tasks/networking/netmiko_file_transfer.py @@ -22,7 +22,7 @@ def netmiko_file_transfer( * changed (``bool``): the destination file was changed """ - net_connect = task.host.get_connection("netmiko") + net_connect = task.host.get_connection("netmiko", task.nornir.config) kwargs.setdefault("direction", "put") scp_result = file_transfer( net_connect, source_file=source_file, dest_file=dest_file, **kwargs diff --git a/nornir/plugins/tasks/networking/netmiko_send_command.py b/nornir/plugins/tasks/networking/netmiko_send_command.py index 3a2872f4..5165a884 100644 --- a/nornir/plugins/tasks/networking/netmiko_send_command.py +++ b/nornir/plugins/tasks/networking/netmiko_send_command.py @@ -18,7 +18,7 @@ def netmiko_send_command( Result object with the following attributes set: * result (``dict``): dictionary with the result of the show command. """ - net_connect = task.host.get_connection("netmiko") + net_connect = task.host.get_connection("netmiko", task.nornir.config) if use_timing: result = net_connect.send_command_timing(command_string, **kwargs) else: diff --git a/nornir/plugins/tasks/networking/netmiko_send_config.py b/nornir/plugins/tasks/networking/netmiko_send_config.py index 0d423332..3876e64e 100644 --- a/nornir/plugins/tasks/networking/netmiko_send_config.py +++ b/nornir/plugins/tasks/networking/netmiko_send_config.py @@ -21,7 +21,7 @@ def netmiko_send_config( Result object with the following attributes set: * result (``dict``): dictionary showing the CLI from the configuration changes """ - net_connect = task.host.get_connection("netmiko") + net_connect = task.host.get_connection("netmiko", task.nornir.config) net_connect.enable() if config_commands: result = net_connect.send_config_set(config_commands=config_commands, **kwargs) diff --git a/tests/core/test_connections.py b/tests/core/test_connections.py index 10ba1f75..4674ce10 100644 --- a/tests/core/test_connections.py +++ b/tests/core/test_connections.py @@ -43,17 +43,17 @@ class AnotherDummyConnectionPlugin(DummyConnectionPlugin): def open_and_close_connection(task): - task.host.open_connection("dummy") + task.host.open_connection("dummy", task.nornir.config) assert "dummy" in task.host.connections task.host.close_connection("dummy") assert "dummy" not in task.host.connections def open_connection_twice(task): - task.host.open_connection("dummy") + task.host.open_connection("dummy", task.nornir.config) assert "dummy" in task.host.connections try: - task.host.open_connection("dummy") + task.host.open_connection("dummy", task.nornir.config) raise Exception("I shouldn't make it here") except ConnectionAlreadyOpen: task.host.close_connection("dummy") @@ -70,11 +70,11 @@ def close_not_opened_connection(task): def a_task(task): - task.host.get_connection("dummy") + task.host.get_connection("dummy", task.nornir.config) def validate_params(task, conn, params): - task.host.get_connection(conn) + task.host.get_connection(conn, task.nornir.config) for k, v in params.items(): assert getattr(task.host.connections[conn], k) == v diff --git a/tests/plugins/functions/text/test_print_result.py b/tests/plugins/functions/text/test_print_result.py index 7dbd3403..b7e70ed3 100644 --- a/tests/plugins/functions/text/test_print_result.py +++ b/tests/plugins/functions/text/test_print_result.py @@ -1,9 +1,10 @@ -import os import logging +import os +from nornir.core.task import Result from nornir.plugins.functions.text import print_result from nornir.plugins.functions.text import print_title -from nornir.core.task import Result + from tests.wrapper import wrap_cli_test output_dir = "{}/output_data".format(os.path.dirname(os.path.realpath(__file__))) diff --git a/tests/plugins/tasks/networking/test_napalm_cli.py b/tests/plugins/tasks/networking/test_napalm_cli.py index 4db1ae82..da9f13b2 100644 --- a/tests/plugins/tasks/networking/test_napalm_cli.py +++ b/tests/plugins/tasks/networking/test_napalm_cli.py @@ -15,7 +15,10 @@ def connect(task, extras): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", extras={"optional_args": extras}, default_to_host_attributes=True + "napalm", + task.nornir.config, + extras={"optional_args": extras}, + default_to_host_attributes=True, ) diff --git a/tests/plugins/tasks/networking/test_napalm_configure.py b/tests/plugins/tasks/networking/test_napalm_configure.py index c87079fc..805ff309 100644 --- a/tests/plugins/tasks/networking/test_napalm_configure.py +++ b/tests/plugins/tasks/networking/test_napalm_configure.py @@ -12,7 +12,10 @@ def connect(task, extras): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", extras={"optional_args": extras}, default_to_host_attributes=True + "napalm", + task.nornir.config, + extras={"optional_args": extras}, + default_to_host_attributes=True, ) diff --git a/tests/plugins/tasks/networking/test_napalm_get.py b/tests/plugins/tasks/networking/test_napalm_get.py index 0c4f9f03..559cf753 100644 --- a/tests/plugins/tasks/networking/test_napalm_get.py +++ b/tests/plugins/tasks/networking/test_napalm_get.py @@ -10,7 +10,10 @@ def connect(task, extras): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", extras={"optional_args": extras}, default_to_host_attributes=True + "napalm", + task.nornir.config, + extras={"optional_args": extras}, + default_to_host_attributes=True, ) diff --git a/tests/plugins/tasks/networking/test_napalm_validate.py b/tests/plugins/tasks/networking/test_napalm_validate.py index 99bdbe7c..c0eca8a8 100644 --- a/tests/plugins/tasks/networking/test_napalm_validate.py +++ b/tests/plugins/tasks/networking/test_napalm_validate.py @@ -10,7 +10,10 @@ def connect(task, extras): if "napalm" in task.host.connections: task.host.close_connection("napalm") task.host.open_connection( - "napalm", extras={"optional_args": extras}, default_to_host_attributes=True + "napalm", + task.nornir.config, + extras={"optional_args": extras}, + default_to_host_attributes=True, ) From 5a6bb1a1bb29c451f30920d879dc542fc1d53e85 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Wed, 24 Oct 2018 15:06:15 +0200 Subject: [PATCH 087/109] fix documentation --- docs/howto/handling_connections.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/howto/handling_connections.ipynb b/docs/howto/handling_connections.ipynb index e8986ef9..a04f1da9 100644 --- a/docs/howto/handling_connections.ipynb +++ b/docs/howto/handling_connections.ipynb @@ -103,7 +103,7 @@ ], "source": [ "def task_manages_connection_manually(task):\n", - " task.host.open_connection(\"napalm\")\n", + " task.host.open_connection(\"napalm\", configuration=task.nornir.config)\n", " r = task.run(\n", " task=napalm_get,\n", " getters=[\"facts\"]\n", From 8056a321f592b43a79c0fd37bb3f1e8661c9d78d Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sat, 27 Oct 2018 15:06:15 +0200 Subject: [PATCH 088/109] added children_of_group method to inventory --- nornir/core/inventory.py | 18 ++++++++++++----- tests/core/test_inventory.py | 38 ++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index cb857c6e..8e719b7b 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -1,5 +1,5 @@ from collections import UserList -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Set, Union from nornir.core.configuration import Config from nornir.core.connections import ConnectionPlugin, Connections @@ -53,10 +53,7 @@ def __init__(self, *args, **kwargs) -> None: self.refs: List["Group"] = kwargs.get("refs", []) def __contains__(self, value) -> bool: - if isinstance(value, str): - return value in self.data - else: - return value in self.refs + return value in self.data or value in self.refs class InventoryElement(BaseAttributes): @@ -423,3 +420,14 @@ def filter(self, filter_obj=None, filter_func=None, *args, **kwargs): def __len__(self): return self.hosts.__len__() + + def children_of_group(self, group: Union[str, Group]) -> Set[Host]: + """ + Returns set of hosts that belongs to a group including those that belong + indirectly via inheritance + """ + hosts: List[Host] = set() + for host in self.hosts.values(): + if host.has_parent_group(group): + hosts.add(host) + return hosts diff --git a/tests/core/test_inventory.py b/tests/core/test_inventory.py index 9374eb99..8b3ee18a 100644 --- a/tests/core/test_inventory.py +++ b/tests/core/test_inventory.py @@ -197,3 +197,41 @@ def test_defaults(self): assert inv.hosts["dev3.group_2"].password == "asd" assert inv.hosts["dev4.group_2"].password == "from_parent_group" assert inv.hosts["dev5.no_group"].password == "asd" + + def test_children_of_str(self): + inv = deserializer.Inventory.deserialize(**inv_dict) + assert inv.children_of_group("parent_group") == { + inv.hosts["dev1.group_1"], + inv.hosts["dev2.group_1"], + inv.hosts["dev4.group_2"], + } + + assert inv.children_of_group("group_1") == { + inv.hosts["dev1.group_1"], + inv.hosts["dev2.group_1"], + } + + assert inv.children_of_group("group_2") == { + inv.hosts["dev4.group_2"], + inv.hosts["dev3.group_2"], + } + + assert inv.children_of_group("blah") == set() + + def test_children_of_obj(self): + inv = deserializer.Inventory.deserialize(**inv_dict) + assert inv.children_of_group(inv.groups["parent_group"]) == { + inv.hosts["dev1.group_1"], + inv.hosts["dev2.group_1"], + inv.hosts["dev4.group_2"], + } + + assert inv.children_of_group(inv.groups["group_1"]) == { + inv.hosts["dev1.group_1"], + inv.hosts["dev2.group_1"], + } + + assert inv.children_of_group(inv.groups["group_2"]) == { + inv.hosts["dev4.group_2"], + inv.hosts["dev3.group_2"], + } From 916b3b436ecfe32b750b285a237c16941253ce3b Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sat, 27 Oct 2018 16:39:56 +0200 Subject: [PATCH 089/109] bugfix: globalstate wasn't being handled properly --- nornir/core/state.py | 8 +++++--- nornir/init_nornir.py | 4 ++-- tests/conftest.py | 5 +++-- tests/core/test_InitNornir.py | 14 ++++++-------- tests/core/test_tasks.py | 22 ++++++++++++++++++++++ 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/nornir/core/state.py b/nornir/core/state.py index 311b2633..e55613a3 100644 --- a/nornir/core/state.py +++ b/nornir/core/state.py @@ -7,9 +7,11 @@ class GlobalState(object): failed_hosts: Hosts that have failed to run a task properly """ - def __init__(self): - self.dry_run = None - self.failed_hosts = set() + __slots__ = "dry_run", "failed_hosts" + + def __init__(self, dry_run=None, failed_hosts=None): + self.dry_run = dry_run + self.failed_hosts = failed_hosts or set() def recover_host(self, host): """Remove ``host`` from list of failed hosts.""" diff --git a/nornir/init_nornir.py b/nornir/init_nornir.py index caf71a1c..1779aff7 100644 --- a/nornir/init_nornir.py +++ b/nornir/init_nornir.py @@ -42,7 +42,7 @@ def InitNornir(config_file="", dry_run=False, configure_logging=True, **kwargs): conf = Config.load_from_file(config_file, **kwargs) - GlobalState.dry_run = dry_run + data = GlobalState(dry_run=dry_run) if configure_logging: conf.logging.configure() @@ -53,4 +53,4 @@ def InitNornir(config_file="", dry_run=False, configure_logging=True, **kwargs): **conf.inventory.options, ) - return Nornir(inventory=inv, config=conf) + return Nornir(inventory=inv, config=conf, data=data) diff --git a/tests/conftest.py b/tests/conftest.py index ee9618d5..97199233 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ import pytest -global_data = GlobalState() +global_data = GlobalState(dry_run=True) logging.basicConfig( @@ -63,5 +63,6 @@ def nornir(request): @pytest.fixture(scope="function", autouse=True) -def reset_failed_hosts(): +def reset_data(): + global_data.dry_run = True global_data.reset_failed_hosts() diff --git a/tests/core/test_InitNornir.py b/tests/core/test_InitNornir.py index 85bbf6b4..b02d17b2 100644 --- a/tests/core/test_InitNornir.py +++ b/tests/core/test_InitNornir.py @@ -21,18 +21,16 @@ def __init__(self, **kwargs): class Test(object): def test_InitNornir_defaults(self): os.chdir("tests/inventory_data/") - try: - nr = InitNornir() - finally: - os.chdir("../../") - assert not nr.state.dry_run + nr = InitNornir() + os.chdir("../../") + assert not nr.data.dry_run assert nr.config.core.num_workers == 20 assert len(nr.inventory.hosts) assert len(nr.inventory.groups) def test_InitNornir_file(self): nr = InitNornir(config_file=os.path.join(dir_path, "a_config.yaml")) - assert not nr.state.dry_run + assert not nr.data.dry_run assert nr.config.core.num_workers == 100 assert len(nr.inventory.hosts) assert len(nr.inventory.groups) @@ -48,7 +46,7 @@ def test_InitNornir_programmatically(self): }, }, ) - assert not nr.state.dry_run + assert not nr.data.dry_run assert nr.config.core.num_workers == 100 assert len(nr.inventory.hosts) assert len(nr.inventory.groups) @@ -58,7 +56,7 @@ def test_InitNornir_combined(self): config_file=os.path.join(dir_path, "a_config.yaml"), core={"num_workers": 200}, ) - assert not nr.state.dry_run + assert not nr.data.dry_run assert nr.config.core.num_workers == 200 assert len(nr.inventory.hosts) assert len(nr.inventory.groups) diff --git a/tests/core/test_tasks.py b/tests/core/test_tasks.py index 4080ba48..f9de585f 100644 --- a/tests/core/test_tasks.py +++ b/tests/core/test_tasks.py @@ -3,6 +3,10 @@ from nornir.plugins.tasks import commands +def a_task_to_test_dry_run(task, expected_dry_run_value, dry_run=None): + assert task.is_dry_run(dry_run) is expected_dry_run_value + + def task_fails_for_some(task): if task.host.name == "dev3.group_2": # let's hardcode a failure @@ -94,3 +98,21 @@ def test_severity(self, nornir): else: assert result[0].severity_level == logging.WARN assert result[1].severity_level == logging.DEBUG + + def test_dry_run(self, nornir): + host = nornir.filter(name="dev3.group_2") + r = host.run(a_task_to_test_dry_run, expected_dry_run_value=True) + assert not r["dev3.group_2"].failed + + r = host.run( + a_task_to_test_dry_run, dry_run=False, expected_dry_run_value=False + ) + assert not r["dev3.group_2"].failed + + nornir.data.dry_run = False + r = host.run(a_task_to_test_dry_run, expected_dry_run_value=False) + assert not r["dev3.group_2"].failed + + nornir.data.dry_run = True + r = host.run(a_task_to_test_dry_run, expected_dry_run_value=False) + assert r["dev3.group_2"].failed From 07191d5441bba0601b8e621cc8f406afc511e435 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sat, 27 Oct 2018 17:09:53 +0200 Subject: [PATCH 090/109] updated tutorial for nornir 2.0.0 --- docs/howto/inventory.ipynb | 1549 ----------------- docs/howto/inventory/groups.yaml | 21 - docs/howto/inventory/hosts.yaml | 162 -- docs/requirements.txt | 2 +- docs/tutorials/intro/config.yaml | 14 +- docs/tutorials/intro/executing_tasks.ipynb | 26 +- docs/tutorials/intro/failed_tasks.ipynb | 38 +- docs/tutorials/intro/grouping_tasks.ipynb | 2 +- .../tutorials/intro/initializing_nornir.ipynb | 38 +- docs/tutorials/intro/inventory.ipynb | 729 +++++--- .../intro}/inventory/defaults.yaml | 1 - docs/tutorials/intro/inventory/groups.yaml | 20 +- docs/tutorials/intro/inventory/hosts.yaml | 132 +- docs/tutorials/intro/task_results.ipynb | 2 +- docs/tutorials/intro/templates/eos/base.j2 | 2 +- docs/tutorials/intro/templates/junos/base.j2 | 2 +- setup.py | 2 +- tox.ini | 2 + 18 files changed, 632 insertions(+), 2112 deletions(-) delete mode 100644 docs/howto/inventory.ipynb delete mode 100644 docs/howto/inventory/groups.yaml delete mode 100644 docs/howto/inventory/hosts.yaml rename docs/{howto => tutorials/intro}/inventory/defaults.yaml (67%) diff --git a/docs/howto/inventory.ipynb b/docs/howto/inventory.ipynb deleted file mode 100644 index 3ca5b21c..00000000 --- a/docs/howto/inventory.ipynb +++ /dev/null @@ -1,1549 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "nbsphinx": "hidden" - }, - "outputs": [], - "source": [ - "# ignore this cell, this is just a helper cell to provide the magic %highlight_file\n", - "%run ../highlighter.py" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Inventory\n", - "\n", - "The Inventory is arguably the most important piece of nornir. Let's see how it works. To begin with the [inventory](../ref/api/inventory.rst#nornir.core.inventory.Inventory) is comprised of [hosts](../ref/api/inventory.rst#nornir.core.inventory.Host), [groups](../ref/api/inventory.rst#nornir.core.inventory.Group) and [defaults](../ref/api/inventory.rst#nornir.core.inventory.Defaults).\n", - "\n", - "In this tutorial we are using the [SimpleInventory](../plugins/inventory/simple.rst#nornir.plugins.inventory.simple.SimpleInventory) plugin. This inventory plugin stores all the relevant data in three files. Let's start by checking them:" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
  1 ---\n",
-       "  2 host1.cmh:\n",
-       "  3     hostname: 127.0.0.1\n",
-       "  4     port: 2201\n",
-       "  5     username: vagrant\n",
-       "  6     password: vagrant\n",
-       "  7     groups:\n",
-       "  8         - cmh\n",
-       "  9     platform: linux\n",
-       " 10     data:\n",
-       " 11         site: cmh\n",
-       " 12         role: host\n",
-       " 13         type: host\n",
-       " 14         nested_data:\n",
-       " 15             a_dict:\n",
-       " 16                 a: 1\n",
-       " 17                 b: 2\n",
-       " 18             a_list: [1, 2]\n",
-       " 19             a_string: "asdasd"\n",
-       " 20 \n",
-       " 21 host2.cmh:\n",
-       " 22     hostname: 127.0.0.1\n",
-       " 23     port: 2202\n",
-       " 24     username: vagrant\n",
-       " 25     password: vagrant\n",
-       " 26     groups:\n",
-       " 27         - cmh\n",
-       " 28     platform: linux\n",
-       " 29     data:\n",
-       " 30         site: cmh\n",
-       " 31         role: host\n",
-       " 32         type: host\n",
-       " 33         nested_data:\n",
-       " 34             a_dict:\n",
-       " 35                 b: 2\n",
-       " 36                 c: 3\n",
-       " 37             a_list: [1, 2]\n",
-       " 38             a_string: "qwe"\n",
-       " 39 \n",
-       " 40 spine00.cmh:\n",
-       " 41     hostname: 127.0.0.1\n",
-       " 42     username: vagrant\n",
-       " 43     password: vagrant\n",
-       " 44     port: 12444\n",
-       " 45     platform: eos\n",
-       " 46     groups:\n",
-       " 47         - cmh\n",
-       " 48     data:\n",
-       " 49         site: cmh\n",
-       " 50         role: spine\n",
-       " 51         type: network_device\n",
-       " 52 \n",
-       " 53 spine01.cmh:\n",
-       " 54     hostname: 127.0.0.1\n",
-       " 55     username: vagrant\n",
-       " 56     password: ""\n",
-       " 57     port: 12204\n",
-       " 58     platform: junos\n",
-       " 59     groups:\n",
-       " 60         - cmh\n",
-       " 61     data:\n",
-       " 62         site: cmh\n",
-       " 63         role: spine\n",
-       " 64         type: network_device\n",
-       " 65 \n",
-       " 66 leaf00.cmh:\n",
-       " 67     hostname: 127.0.0.1\n",
-       " 68     username: vagrant\n",
-       " 69     password: vagrant\n",
-       " 70     port: 12443\n",
-       " 71     groups:\n",
-       " 72         - cmh\n",
-       " 73     platform: eos\n",
-       " 74     data:\n",
-       " 75         site: cmh\n",
-       " 76         role: leaf\n",
-       " 77         type: network_device\n",
-       " 78         asn: 65100\n",
-       " 79 \n",
-       " 80 leaf01.cmh:\n",
-       " 81     hostname: 127.0.0.1\n",
-       " 82     username: vagrant\n",
-       " 83     password: ""\n",
-       " 84     port: 12203\n",
-       " 85     groups:\n",
-       " 86         - cmh\n",
-       " 87     platform: junos\n",
-       " 88     data:\n",
-       " 89         site: cmh\n",
-       " 90         role: leaf\n",
-       " 91         type: network_device\n",
-       " 92         asn: 65101\n",
-       " 93 \n",
-       " 94 host1.bma:\n",
-       " 95     groups:\n",
-       " 96         - bma\n",
-       " 97     platform: linux\n",
-       " 98     data:\n",
-       " 99         type: host\n",
-       "100         site: bma\n",
-       "101         role: host\n",
-       "102 \n",
-       "103 host2.bma:\n",
-       "104     groups:\n",
-       "105         - bma\n",
-       "106     platform: linux\n",
-       "107     data:\n",
-       "108         type: host\n",
-       "109         site: bma\n",
-       "110         role: host\n",
-       "111 \n",
-       "112 spine00.bma:\n",
-       "113     hostname: 127.0.0.1\n",
-       "114     username: vagrant\n",
-       "115     password: vagrant\n",
-       "116     port: 12444\n",
-       "117     groups:\n",
-       "118         - bma\n",
-       "119     platform: eos\n",
-       "120     data:\n",
-       "121         site: bma\n",
-       "122         role: spine\n",
-       "123         type: network_device\n",
-       "124 \n",
-       "125 spine01.bma:\n",
-       "126     hostname: 127.0.0.1\n",
-       "127     username: vagrant\n",
-       "128     password: ""\n",
-       "129     port: 12204\n",
-       "130     groups:\n",
-       "131         - bma\n",
-       "132     platform: junos\n",
-       "133     data:\n",
-       "134         type: network_device\n",
-       "135         site: bma\n",
-       "136         role: spine\n",
-       "137 \n",
-       "138 leaf00.bma:\n",
-       "139     hostname: 127.0.0.1\n",
-       "140     username: vagrant\n",
-       "141     password: vagrant\n",
-       "142     port: 12443\n",
-       "143     groups:\n",
-       "144         - bma\n",
-       "145     platform: eos\n",
-       "146     data:\n",
-       "147         type: network_device\n",
-       "148         site: bma\n",
-       "149         role: leaf\n",
-       "150 \n",
-       "151 leaf01.bma:\n",
-       "152     hostname: 127.0.0.1\n",
-       "153     username: vagrant\n",
-       "154     password: wrong_password\n",
-       "155     port: 12203\n",
-       "156     groups:\n",
-       "157         - bma\n",
-       "158     platform: junos\n",
-       "159     data:\n",
-       "160         type: network_device\n",
-       "161         site: bma\n",
-       "162         role: leaf\n",
-       "
\n", - "\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# hosts file\n", - "%highlight_file inventory/hosts.yaml" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The hosts file is basically a map where the outermost key is the name of the host and then an `InventoryElement` object. You can see the schema of the object by executing:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"title\": \"InventoryElement\",\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"hostname\": {\n", - " \"title\": \"Hostname\",\n", - " \"required\": false,\n", - " \"type\": \"str\"\n", - " },\n", - " \"port\": {\n", - " \"title\": \"Port\",\n", - " \"required\": false,\n", - " \"type\": \"int\"\n", - " },\n", - " \"username\": {\n", - " \"title\": \"Username\",\n", - " \"required\": false,\n", - " \"type\": \"str\"\n", - " },\n", - " \"password\": {\n", - " \"title\": \"Password\",\n", - " \"required\": false,\n", - " \"type\": \"str\"\n", - " },\n", - " \"platform\": {\n", - " \"title\": \"Platform\",\n", - " \"required\": false,\n", - " \"type\": \"str\"\n", - " },\n", - " \"groups\": {\n", - " \"title\": \"Groups\",\n", - " \"required\": false,\n", - " \"default\": [],\n", - " \"type\": \"list\",\n", - " \"item_type\": \"str\"\n", - " },\n", - " \"data\": {\n", - " \"title\": \"Data\",\n", - " \"required\": false,\n", - " \"default\": {},\n", - " \"type\": \"mapping\",\n", - " \"item_type\": \"any\",\n", - " \"key_type\": \"str\"\n", - " },\n", - " \"connection_options\": {\n", - " \"title\": \"Connection_Options\",\n", - " \"required\": false,\n", - " \"default\": {},\n", - " \"type\": \"mapping\",\n", - " \"item_type\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"hostname\": {\n", - " \"title\": \"Hostname\",\n", - " \"required\": false,\n", - " \"type\": \"str\"\n", - " },\n", - " \"port\": {\n", - " \"title\": \"Port\",\n", - " \"required\": false,\n", - " \"type\": \"int\"\n", - " },\n", - " \"username\": {\n", - " \"title\": \"Username\",\n", - " \"required\": false,\n", - " \"type\": \"str\"\n", - " },\n", - " \"password\": {\n", - " \"title\": \"Password\",\n", - " \"required\": false,\n", - " \"type\": \"str\"\n", - " },\n", - " \"platform\": {\n", - " \"title\": \"Platform\",\n", - " \"required\": false,\n", - " \"type\": \"str\"\n", - " },\n", - " \"extras\": {\n", - " \"title\": \"Extras\",\n", - " \"required\": false,\n", - " \"type\": \"mapping\",\n", - " \"item_type\": \"any\",\n", - " \"key_type\": \"str\"\n", - " }\n", - " }\n", - " },\n", - " \"key_type\": \"str\"\n", - " }\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "from nornir.core.deserializer.inventory import InventoryElement\n", - "import json\n", - "print(json.dumps(InventoryElement.schema(), indent=4))" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
 1 ---\n",
-       " 2 global:\n",
-       " 3     data:\n",
-       " 4         domain: global.local\n",
-       " 5         asn: 1\n",
-       " 6 \n",
-       " 7 eu:\n",
-       " 8     data:\n",
-       " 9         asn: 65100\n",
-       "10 \n",
-       "11 bma:\n",
-       "12     groups:\n",
-       "13         - eu\n",
-       "14         - global\n",
-       "15 \n",
-       "16 cmh:\n",
-       "17     data:\n",
-       "18         asn: 65000\n",
-       "19         vlans:\n",
-       "20           100: frontend\n",
-       "21           200: backend\n",
-       "
\n", - "\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# groups file\n", - "%highlight_file inventory/groups.yaml" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The groups file works the same way as the hosts file." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
1 ---\n",
-       "2 username: admin\n",
-       "3 data:\n",
-       "4     domain: acme.local\n",
-       "
\n", - "\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# defaults\n", - "%highlight_file inventory/defaults.yaml" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finally, the defaults file has the same schema as the ``InventoryElement`` we described before. We will see how the data in the groups and defaults file is used later on in this tutorial." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Pretty similar to the hosts file.\n", - "\n", - "### Accessing the inventory\n", - "\n", - "You can access the [inventory](../ref/api/inventory.rst#nornir.core.inventory.Inventory) with the `inventory` attribute:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "from nornir import InitNornir\n", - "nr = InitNornir(\n", - " inventory={\n", - " \"options\": {\n", - " \"host_file\": \"inventory/hosts.yaml\",\n", - " \"group_file\": \"inventory/groups.yaml\",\n", - " \"defaults_file\": \"inventory/defaults.yaml\",\n", - " }\n", - " }\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The inventory has two dict-like attributes `hosts` and `groups` that you can use to access the hosts and groups respectively:" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'host1.cmh': Host: host1.cmh,\n", - " 'host2.cmh': Host: host2.cmh,\n", - " 'spine00.cmh': Host: spine00.cmh,\n", - " 'spine01.cmh': Host: spine01.cmh,\n", - " 'leaf00.cmh': Host: leaf00.cmh,\n", - " 'leaf01.cmh': Host: leaf01.cmh,\n", - " 'host1.bma': Host: host1.bma,\n", - " 'host2.bma': Host: host2.bma,\n", - " 'spine00.bma': Host: spine00.bma,\n", - " 'spine01.bma': Host: spine01.bma,\n", - " 'leaf00.bma': Host: leaf00.bma,\n", - " 'leaf01.bma': Host: leaf01.bma}" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "nr.inventory.hosts" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'global': Group: global,\n", - " 'eu': Group: eu,\n", - " 'bma': Group: bma,\n", - " 'cmh': Group: cmh}" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "nr.inventory.groups" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Host: leaf01.bma" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "nr.inventory.hosts[\"leaf01.bma\"]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Hosts and groups are also dict-like objects:" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "dict_keys(['type', 'site', 'role', 'asn', 'domain'])" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "host = nr.inventory.hosts[\"leaf01.bma\"]\n", - "host.keys()" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'bma'" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "host[\"site\"]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Inheritance model\n", - "\n", - "Let's see how the inheritance models works by example. Let's start by looking again at the groups file:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
 1 ---\n",
-       " 2 global:\n",
-       " 3     data:\n",
-       " 4         domain: global.local\n",
-       " 5         asn: 1\n",
-       " 6 \n",
-       " 7 eu:\n",
-       " 8     data:\n",
-       " 9         asn: 65100\n",
-       "10 \n",
-       "11 bma:\n",
-       "12     groups:\n",
-       "13         - eu\n",
-       "14         - global\n",
-       "15 \n",
-       "16 cmh:\n",
-       "17     data:\n",
-       "18         asn: 65000\n",
-       "19         vlans:\n",
-       "20           100: frontend\n",
-       "21           200: backend\n",
-       "
\n", - "\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# groups file\n", - "%highlight_file inventory/groups.yaml" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
1 ---\n",
-       "2 username: admin\n",
-       "3 data:\n",
-       "4     domain: acme.local\n",
-       "
\n", - "\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# defaults file\n", - "%highlight_file inventory/defaults.yaml" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The host `leaf01.bma` belongs to the group `bma` which in turn belongs to the groups `eu` and `global`. The host `spine00.cmh` belongs to the group `cmh` which doesn't belong to any other group.\n", - "\n", - "Data resolution works by iterating recursively over all the parent groups and trying to see if that parent group (or any of it's parents) contains the data. For instance:" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'global.local'" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "leaf01_bma = nr.inventory.hosts[\"leaf01.bma\"]\n", - "leaf01_bma[\"domain\"] # comes from the group `global`" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "65100" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "leaf01_bma[\"asn\"] # comes from group `eu`" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finally, the `defaults` file contains data that will be returned if neither the host nor the parents have a specific value for it." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'acme.local'" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "leaf01_cmh = nr.inventory.hosts[\"leaf01.cmh\"]\n", - "leaf01_cmh[\"domain\"] # comes from defaults" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If nornir can't resolve the data you should get a KeyError as usual:" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Couldn't find key: 'non_existent'\n" - ] - } - ], - "source": [ - "try:\n", - " leaf01_cmh[\"non_existent\"]\n", - "except KeyError as e:\n", - " print(f\"Couldn't find key: {e}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can also try to access data without recursive resolution by using the `data` attribute. For example, if we try to access `leaf01_cmh.data[\"domain\"]` we should get an error as the host itself doesn't have that data:" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Couldn't find key: 'domain'\n" - ] - } - ], - "source": [ - "try:\n", - " leaf01_cmh.data[\"domain\"]\n", - "except KeyError as e:\n", - " print(f\"Couldn't find key: {e}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Filtering the inventory\n", - "\n", - "So far we have seen that `nr.inventory.hosts` and `nr.inventory.groups` are dict-like objects that we can use to iterate over all the hosts and groups or to access any particular one directly. Now we are going to see how we can do some fancy filtering that will enable us to operate on groups of hosts based on their properties.\n", - "\n", - "The simpler way of filtering hosts is by `` pairs. For instance:" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "dict_keys(['host1.cmh', 'host2.cmh', 'spine00.cmh', 'spine01.cmh', 'leaf00.cmh', 'leaf01.cmh'])" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "nr.filter(site=\"cmh\").inventory.hosts.keys()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can also filter using multiple `` pairs:" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "dict_keys(['spine00.cmh', 'spine01.cmh'])" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "nr.filter(site=\"cmh\", role=\"spine\").inventory.hosts.keys()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Filter is cumulative:" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "dict_keys(['spine00.cmh', 'spine01.cmh'])" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "nr.filter(site=\"cmh\").filter(role=\"spine\").inventory.hosts.keys()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Or:" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "dict_keys(['spine00.cmh', 'spine01.cmh'])" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cmh = nr.filter(site=\"cmh\")\n", - "cmh.filter(role=\"spine\").inventory.hosts.keys()" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "dict_keys(['leaf00.cmh', 'leaf01.cmh'])" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cmh.filter(role=\"leaf\").inventory.hosts.keys()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can also grab the children of a group:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Advanced filtering\n", - "\n", - "Sometimes you need more fancy filtering. For those cases you have two options:\n", - "\n", - "1. Use a filter function.\n", - "2. Use a filter object.\n", - "\n", - "##### Filter functions\n", - "\n", - "The ``filter_func`` parameter let's you run your own code to filter the hosts. The function signature is as simple as ``my_func(host)`` where host is an object of type [Host](../ref/api/inventory.rst#nornir.core.inventory.Host) and it has to return either ``True`` or ``False`` to indicate if you want to host or not." - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "dict_keys(['spine00.cmh', 'spine01.cmh', 'spine00.bma', 'spine01.bma'])" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "def has_long_name(host):\n", - " return len(host.name) == 11\n", - "\n", - "nr.filter(filter_func=has_long_name).inventory.hosts.keys()" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "dict_keys(['host1.cmh', 'host2.cmh', 'host1.bma', 'host2.bma'])" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Or a lambda function\n", - "nr.filter(filter_func=lambda h: len(h.name) == 9).inventory.hosts.keys()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### Filter Object\n", - "\n", - "You can also use a filter object to create incrementally a complex query object. Let's see how it works by example:" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [], - "source": [ - "# first you need to import the F object\n", - "from nornir.core.filter import F" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "dict_keys(['host1.cmh', 'host2.cmh', 'spine00.cmh', 'spine01.cmh', 'leaf00.cmh', 'leaf01.cmh'])\n" - ] - } - ], - "source": [ - "# hosts in group cmh\n", - "cmh = nr.filter(F(groups__contains=\"cmh\"))\n", - "print(cmh.inventory.hosts.keys())" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "dict_keys(['host1.cmh', 'host2.cmh', 'spine00.cmh', 'leaf00.cmh', 'host1.bma', 'host2.bma', 'spine00.bma', 'leaf00.bma'])\n" - ] - } - ], - "source": [ - "# devices running either linux or eos\n", - "linux_or_eos = nr.filter(F(platform=\"linux\") | F(platform=\"eos\"))\n", - "print(linux_or_eos.inventory.hosts.keys())" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "dict_keys(['spine00.cmh', 'spine01.cmh'])\n" - ] - } - ], - "source": [ - "# spines in cmh\n", - "cmh_and_spine = nr.filter(F(groups__contains=\"cmh\") & F(role=\"spine\"))\n", - "print(cmh_and_spine.inventory.hosts.keys())" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "dict_keys(['host1.cmh', 'host2.cmh', 'leaf00.cmh', 'leaf01.cmh'])\n" - ] - } - ], - "source": [ - "# cmh devices that are not spines\n", - "cmh_and_not_spine = nr.filter(F(groups__contains=\"cmh\") & ~F(role=\"spine\"))\n", - "print(cmh_and_not_spine.inventory.hosts.keys())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can also access nested data and even check if dicts/lists/strings contains elements. Again, let's see by example:" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "dict_keys(['host1.cmh'])\n" - ] - } - ], - "source": [ - "nested_string_asd = nr.filter(F(nested_data__a_string__contains=\"asd\"))\n", - "print(nested_string_asd.inventory.hosts.keys())" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "dict_keys(['host2.cmh'])\n" - ] - } - ], - "source": [ - "a_dict_element_equals = nr.filter(F(nested_data__a_dict__c=3))\n", - "print(a_dict_element_equals.inventory.hosts.keys())" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "dict_keys(['host1.cmh', 'host2.cmh'])\n" - ] - } - ], - "source": [ - "a_list_contains = nr.filter(F(nested_data__a_list__contains=2))\n", - "print(a_list_contains.inventory.hosts.keys())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can basically access any nested data by separating the elements in the path with two underscores `__`. Then you can use `__contains` to check if an element exists or if a string has a particular substring." - ] - } - ], - "metadata": { - "celltoolbar": "Edit Metadata", - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/howto/inventory/groups.yaml b/docs/howto/inventory/groups.yaml deleted file mode 100644 index 0a9fc619..00000000 --- a/docs/howto/inventory/groups.yaml +++ /dev/null @@ -1,21 +0,0 @@ ---- -global: - data: - domain: global.local - asn: 1 - -eu: - data: - asn: 65100 - -bma: - groups: - - eu - - global - -cmh: - data: - asn: 65000 - vlans: - 100: frontend - 200: backend diff --git a/docs/howto/inventory/hosts.yaml b/docs/howto/inventory/hosts.yaml deleted file mode 100644 index bc2a039b..00000000 --- a/docs/howto/inventory/hosts.yaml +++ /dev/null @@ -1,162 +0,0 @@ ---- -host1.cmh: - hostname: 127.0.0.1 - port: 2201 - username: vagrant - password: vagrant - groups: - - cmh - platform: linux - data: - site: cmh - role: host - type: host - nested_data: - a_dict: - a: 1 - b: 2 - a_list: [1, 2] - a_string: "asdasd" - -host2.cmh: - hostname: 127.0.0.1 - port: 2202 - username: vagrant - password: vagrant - groups: - - cmh - platform: linux - data: - site: cmh - role: host - type: host - nested_data: - a_dict: - b: 2 - c: 3 - a_list: [1, 2] - a_string: "qwe" - -spine00.cmh: - hostname: 127.0.0.1 - username: vagrant - password: vagrant - port: 12444 - platform: eos - groups: - - cmh - data: - site: cmh - role: spine - type: network_device - -spine01.cmh: - hostname: 127.0.0.1 - username: vagrant - password: "" - port: 12204 - platform: junos - groups: - - cmh - data: - site: cmh - role: spine - type: network_device - -leaf00.cmh: - hostname: 127.0.0.1 - username: vagrant - password: vagrant - port: 12443 - groups: - - cmh - platform: eos - data: - site: cmh - role: leaf - type: network_device - asn: 65100 - -leaf01.cmh: - hostname: 127.0.0.1 - username: vagrant - password: "" - port: 12203 - groups: - - cmh - platform: junos - data: - site: cmh - role: leaf - type: network_device - asn: 65101 - -host1.bma: - groups: - - bma - platform: linux - data: - type: host - site: bma - role: host - -host2.bma: - groups: - - bma - platform: linux - data: - type: host - site: bma - role: host - -spine00.bma: - hostname: 127.0.0.1 - username: vagrant - password: vagrant - port: 12444 - groups: - - bma - platform: eos - data: - site: bma - role: spine - type: network_device - -spine01.bma: - hostname: 127.0.0.1 - username: vagrant - password: "" - port: 12204 - groups: - - bma - platform: junos - data: - type: network_device - site: bma - role: spine - -leaf00.bma: - hostname: 127.0.0.1 - username: vagrant - password: vagrant - port: 12443 - groups: - - bma - platform: eos - data: - type: network_device - site: bma - role: leaf - -leaf01.bma: - hostname: 127.0.0.1 - username: vagrant - password: wrong_password - port: 12203 - groups: - - bma - platform: junos - data: - type: network_device - site: bma - role: leaf diff --git a/docs/requirements.txt b/docs/requirements.txt index 97828106..e28da7c4 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -sphinx<1.6 +sphinx sphinx_rtd_theme sphinxcontrib-napoleon jupyter diff --git a/docs/tutorials/intro/config.yaml b/docs/tutorials/intro/config.yaml index 0b6eab1a..ab9e0684 100644 --- a/docs/tutorials/intro/config.yaml +++ b/docs/tutorials/intro/config.yaml @@ -1,6 +1,10 @@ --- -num_workers: 100 -inventory: nornir.plugins.inventory.simple.SimpleInventory -SimpleInventory: - host_file: "inventory/hosts.yaml" - group_file: "inventory/groups.yaml" +core: + num_workers: 100 + +inventory: + plugin: nornir.plugins.inventory.simple.SimpleInventory + options: + host_file: "inventory/hosts.yaml" + group_file: "inventory/groups.yaml" + defaults_file: "inventory/defaults.yaml" diff --git a/docs/tutorials/intro/executing_tasks.ipynb b/docs/tutorials/intro/executing_tasks.ipynb index 5928880e..4a073524 100644 --- a/docs/tutorials/intro/executing_tasks.ipynb +++ b/docs/tutorials/intro/executing_tasks.ipynb @@ -36,15 +36,15 @@ "\u001b[0m\u001b[1m\u001b[34m* host1.cmh ** changed : False *************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv remote_command ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0mtotal 8\n", - "drwxrwxrwt 2 root root 4096 Aug 10 11:47 .\n", - "drwxr-xr-x 24 root root 4096 Aug 10 11:47 ..\n", + "drwxrwxrwt 2 root root 4096 Oct 27 14:53 .\n", + "drwxr-xr-x 24 root root 4096 Oct 27 14:53 ..\n", "\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END remote_command ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[34m* host2.cmh ** changed : False *************************************************\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32mvvvv remote_command ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", "\u001b[0mtotal 8\n", - "drwxrwxrwt 2 root root 4096 Aug 10 11:48 .\n", - "drwxr-xr-x 24 root root 4096 Aug 10 11:48 ..\n", + "drwxrwxrwt 2 root root 4096 Oct 27 14:54 .\n", + "drwxr-xr-x 24 root root 4096 Oct 27 14:54 ..\n", "\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END remote_command ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", "\u001b[0m" @@ -93,7 +93,7 @@ " \u001b[0m'model'\u001b[0m: \u001b[0m'vEOS'\u001b[0m,\n", " \u001b[0m'os_version'\u001b[0m: \u001b[0m'4.20.1F-6820520.4201F'\u001b[0m,\n", " \u001b[0m'serial_number'\u001b[0m: \u001b[0m''\u001b[0m,\n", - " \u001b[0m'uptime'\u001b[0m: \u001b[0m77764\u001b[0m,\n", + " \u001b[0m'uptime'\u001b[0m: \u001b[0m499\u001b[0m,\n", " \u001b[0m'vendor'\u001b[0m: \u001b[0m'Arista'\u001b[0m}\u001b[0m}\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END napalm_get ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[34m* spine01.bma ** changed : False ***********************************************\u001b[0m\n", @@ -127,8 +127,8 @@ " \u001b[0m'vlan'\u001b[0m]\u001b[0m,\n", " \u001b[0m'model'\u001b[0m: \u001b[0m'FIREFLY-PERIMETER'\u001b[0m,\n", " \u001b[0m'os_version'\u001b[0m: \u001b[0m'12.1X47-D20.7'\u001b[0m,\n", - " \u001b[0m'serial_number'\u001b[0m: \u001b[0m'7287f12c493d'\u001b[0m,\n", - " \u001b[0m'uptime'\u001b[0m: \u001b[0m77648\u001b[0m,\n", + " \u001b[0m'serial_number'\u001b[0m: \u001b[0m'66c3cbe24e7b'\u001b[0m,\n", + " \u001b[0m'uptime'\u001b[0m: \u001b[0m385\u001b[0m,\n", " \u001b[0m'vendor'\u001b[0m: \u001b[0m'Juniper'\u001b[0m}\u001b[0m}\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END napalm_get ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", "\u001b[0m" @@ -227,12 +227,12 @@ "none 5.0M 0 5.0M 0% /run/lock\n", "none 183M 0 183M 0% /run/shm\n", "/dev/sda1 228M 25M 192M 12% /boot\n", - "vagrant 373G 271G 103G 73% /vagrant\n", + "vagrant 373G 251G 122G 68% /vagrant\n", "\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m---- Available memory ** changed : False --------------------------------------- INFO\u001b[0m\n", "\u001b[0m total used free shared buffers cached\n", - "Mem: 365 87 278 0 8 36\n", - "-/+ buffers/cache: 42 323\n", + "Mem: 365 87 277 0 8 36\n", + "-/+ buffers/cache: 42 322\n", "Swap: 767 0 767\n", "\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m^^^^ END available_resources ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", @@ -246,11 +246,11 @@ "none 5.0M 0 5.0M 0% /run/lock\n", "none 183M 0 183M 0% /run/shm\n", "/dev/sda1 228M 25M 192M 12% /boot\n", - "vagrant 373G 271G 103G 73% /vagrant\n", + "vagrant 373G 251G 122G 68% /vagrant\n", "\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[32m---- Available memory ** changed : False --------------------------------------- INFO\u001b[0m\n", "\u001b[0m total used free shared buffers cached\n", - "Mem: 365 88 277 0 8 36\n", + "Mem: 365 87 277 0 8 36\n", "-/+ buffers/cache: 42 322\n", "Swap: 767 0 767\n", "\u001b[0m\n", @@ -348,7 +348,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.7.0" } }, "nbformat": 4, diff --git a/docs/tutorials/intro/failed_tasks.ipynb b/docs/tutorials/intro/failed_tasks.ipynb index de15cbd7..2cf57cc0 100644 --- a/docs/tutorials/intro/failed_tasks.ipynb +++ b/docs/tutorials/intro/failed_tasks.ipynb @@ -158,27 +158,27 @@ "}\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[31m---- Loading Configuration on the device ** changed : False -------------------- ERROR\u001b[0m\n", "\u001b[0mTraceback (most recent call last):\n", - " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.6/site-packages/napalm/eos/eos.py\", line 231, in _load_config\n", + " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.7/site-packages/napalm/eos/eos.py\", line 231, in _load_config\n", " self.device.run_commands(commands)\n", - " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.6/site-packages/pyeapi/client.py\", line 730, in run_commands\n", + " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.7/site-packages/pyeapi/client.py\", line 730, in run_commands\n", " response = self._connection.execute(commands, encoding, **kwargs)\n", - " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.6/site-packages/pyeapi/eapilib.py\", line 499, in execute\n", + " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.7/site-packages/pyeapi/eapilib.py\", line 499, in execute\n", " response = self.send(request)\n", - " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.6/site-packages/pyeapi/eapilib.py\", line 418, in send\n", + " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.7/site-packages/pyeapi/eapilib.py\", line 418, in send\n", " raise CommandError(code, msg, command_error=err, output=out)\n", "pyeapi.eapilib.CommandError: Error [1002]: CLI command 3 of 6 'system {' failed: invalid command [Invalid input (at token 1: '{')]\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", - " File \"/Users/dbarroso/workspace/nornir/nornir/core/task.py\", line 63, in start\n", + " File \"/Users/dbarroso/workspace/nornir/nornir/core/task.py\", line 62, in start\n", " r = self.task(self, **self.params)\n", " File \"/Users/dbarroso/workspace/nornir/nornir/plugins/tasks/networking/napalm_configure.py\", line 32, in napalm_configure\n", " device.load_merge_candidate(filename=filename, config=configuration)\n", - " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.6/site-packages/napalm/eos/eos.py\", line 245, in load_merge_candidate\n", + " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.7/site-packages/napalm/eos/eos.py\", line 246, in load_merge_candidate\n", " self._load_config(filename, config, False)\n", - " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.6/site-packages/napalm/eos/eos.py\", line 237, in _load_config\n", - " raise MergeConfigException(e.message)\n", + " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.7/site-packages/napalm/eos/eos.py\", line 238, in _load_config\n", + " raise MergeConfigException(msg)\n", "napalm.base.exceptions.MergeConfigException: Error [1002]: CLI command 3 of 6 'system {' failed: invalid command [Invalid input (at token 1: '{')]\n", "\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[31m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", @@ -206,27 +206,27 @@ "}\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[31m---- Loading Configuration on the device ** changed : False -------------------- ERROR\u001b[0m\n", "\u001b[0mTraceback (most recent call last):\n", - " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.6/site-packages/napalm/eos/eos.py\", line 231, in _load_config\n", + " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.7/site-packages/napalm/eos/eos.py\", line 231, in _load_config\n", " self.device.run_commands(commands)\n", - " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.6/site-packages/pyeapi/client.py\", line 730, in run_commands\n", + " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.7/site-packages/pyeapi/client.py\", line 730, in run_commands\n", " response = self._connection.execute(commands, encoding, **kwargs)\n", - " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.6/site-packages/pyeapi/eapilib.py\", line 499, in execute\n", + " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.7/site-packages/pyeapi/eapilib.py\", line 499, in execute\n", " response = self.send(request)\n", - " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.6/site-packages/pyeapi/eapilib.py\", line 418, in send\n", + " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.7/site-packages/pyeapi/eapilib.py\", line 418, in send\n", " raise CommandError(code, msg, command_error=err, output=out)\n", "pyeapi.eapilib.CommandError: Error [1002]: CLI command 3 of 6 'system {' failed: invalid command [Invalid input (at token 1: '{')]\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", - " File \"/Users/dbarroso/workspace/nornir/nornir/core/task.py\", line 63, in start\n", + " File \"/Users/dbarroso/workspace/nornir/nornir/core/task.py\", line 62, in start\n", " r = self.task(self, **self.params)\n", " File \"/Users/dbarroso/workspace/nornir/nornir/plugins/tasks/networking/napalm_configure.py\", line 32, in napalm_configure\n", " device.load_merge_candidate(filename=filename, config=configuration)\n", - " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.6/site-packages/napalm/eos/eos.py\", line 245, in load_merge_candidate\n", + " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.7/site-packages/napalm/eos/eos.py\", line 246, in load_merge_candidate\n", " self._load_config(filename, config, False)\n", - " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.6/site-packages/napalm/eos/eos.py\", line 237, in _load_config\n", - " raise MergeConfigException(e.message)\n", + " File \"/Users/dbarroso/.virtualenvs/nornir/lib/python3.7/site-packages/napalm/eos/eos.py\", line 238, in _load_config\n", + " raise MergeConfigException(msg)\n", "napalm.base.exceptions.MergeConfigException: Error [1002]: CLI command 3 of 6 'system {' failed: invalid command [Invalid input (at token 1: '{')]\n", "\u001b[0m\n", "\u001b[0m\u001b[1m\u001b[31m^^^^ END basic_configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", @@ -471,7 +471,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -484,7 +484,7 @@ } ], "source": [ - "nr = InitNornir(config_file=\"config.yaml\", raise_on_error=True)\n", + "nr = InitNornir(config_file=\"config.yaml\", core={\"raise_on_error\": True})\n", "cmh = nr.filter(site=\"cmh\", type=\"network_device\")\n", "try:\n", " cmh.run(task=basic_configuration)\n", @@ -523,7 +523,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.7.0" } }, "nbformat": 4, diff --git a/docs/tutorials/intro/grouping_tasks.ipynb b/docs/tutorials/intro/grouping_tasks.ipynb index 3f20083a..ecf3b5df 100644 --- a/docs/tutorials/intro/grouping_tasks.ipynb +++ b/docs/tutorials/intro/grouping_tasks.ipynb @@ -374,7 +374,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.7.0" } }, "nbformat": 4, diff --git a/docs/tutorials/intro/initializing_nornir.ipynb b/docs/tutorials/intro/initializing_nornir.ipynb index 7270055c..bbfe2696 100644 --- a/docs/tutorials/intro/initializing_nornir.ipynb +++ b/docs/tutorials/intro/initializing_nornir.ipynb @@ -107,12 +107,16 @@ "}\n", "\n", "\n", - "
1 ---\n",
-       "2 num_workers: 100\n",
-       "3 inventory: nornir.plugins.inventory.simple.SimpleInventory\n",
-       "4 SimpleInventory:\n",
-       "5     host_file: "inventory/hosts.yaml"\n",
-       "6     group_file: "inventory/groups.yaml"\n",
+       "
 1 ---\n",
+       " 2 core:\n",
+       " 3     num_workers: 100\n",
+       " 4 \n",
+       " 5 inventory:\n",
+       " 6     plugin: nornir.plugins.inventory.simple.SimpleInventory\n",
+       " 7     options:\n",
+       " 8         host_file: "inventory/hosts.yaml"\n",
+       " 9         group_file: "inventory/groups.yaml"\n",
+       "10         defaults_file: "inventory/defaults.yaml"\n",
        "
\n",
        "\n"
       ],
@@ -133,10 +137,6 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "
\n",
-    "**Note:** To pass options to the inventory plugin add a top-level dictionary named after the inventory class name; `SimpleInventory` in this example.\n",
-    "
\n",
-    "\n",
     "Now to create the [nornir](../../ref/api/nornir.rst#nornir) object:"
    ]
   },
@@ -164,10 +164,16 @@
    "outputs": [],
    "source": [
     "from nornir import InitNornir\n",
-    "nr = InitNornir(num_workers=100,\n",
-    "                  inventory=\"nornir.plugins.inventory.simple.SimpleInventory\",\n",
-    "                  SimpleInventory={\"host_file\": \"inventory/hosts.yaml\",\n",
-    "                                   \"group_file\": \"inventory/groups.yaml\"})"
+    "nr = InitNornir(\n",
+    "    core={\"num_workers\": 100},\n",
+    "    inventory={\n",
+    "        \"plugin\": \"nornir.plugins.inventory.simple.SimpleInventory\",\n",
+    "        \"options\": {\n",
+    "            \"host_file\": \"inventory/hosts.yaml\",\n",
+    "            \"group_file\": \"inventory/groups.yaml\"\n",
+    "        }\n",
+    "    }\n",
+    ")"
    ]
   },
   {
@@ -184,7 +190,7 @@
    "outputs": [],
    "source": [
     "from nornir import InitNornir\n",
-    "nr = InitNornir(num_workers=50, config_file=\"config.yaml\")"
+    "nr = InitNornir(core={\"num_workers\": 50}, config_file=\"config.yaml\")"
    ]
   },
   {
@@ -224,7 +230,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.5"
+   "version": "3.7.0"
   }
  },
  "nbformat": 4,
diff --git a/docs/tutorials/intro/inventory.ipynb b/docs/tutorials/intro/inventory.ipynb
index 15153ed8..7f46b1c3 100644
--- a/docs/tutorials/intro/inventory.ipynb
+++ b/docs/tutorials/intro/inventory.ipynb
@@ -18,9 +18,9 @@
    "source": [
     "## Inventory\n",
     "\n",
-    "The Inventory is arguably the most important piece of nornir. Let's see how it works. To begin with the [inventory](../../ref/api/inventory.rst#nornir.core.inventory.Inventory) is comprised of [hosts](../../ref/api/inventory.rst#nornir.core.inventory.Host) and [groups](../../ref/api/inventory.rst#nornir.core.inventory.Group).\n",
+    "The Inventory is arguably the most important piece of nornir. Let's see how it works. To begin with the [inventory](../../ref/api/inventory.rst#nornir.core.inventory.Inventory) is comprised of [hosts](../../ref/api/inventory.rst#nornir.core.inventory.Host),  [groups](../../ref/api/inventory.rst#nornir.core.inventory.Group) and [defaults](../../ref/api/inventory.rst#nornir.core.inventory.Defaults).\n",
     "\n",
-    "In this tutorial we are using the [SimpleInventory](../../plugins/inventory/simple.rst#nornir.plugins.inventory.simple.SimpleInventory) plugin. This inventory plugin stores all the relevant data in two files. Let's start by checking them:"
+    "In this tutorial we are using the [SimpleInventory](../../plugins/inventory/simple.rst#nornir.plugins.inventory.simple.SimpleInventory) plugin. This inventory plugin stores all the relevant data in three files. Let’s start by checking them:"
    ]
   },
   {
@@ -111,150 +111,162 @@
        "  4     port: 2201\n",
        "  5     username: vagrant\n",
        "  6     password: vagrant\n",
-       "  7     site: cmh\n",
-       "  8     role: host\n",
-       "  9     groups:\n",
-       " 10         - cmh\n",
-       " 11     platform: linux\n",
-       " 12     type: host\n",
-       " 13     nested_data:\n",
-       " 14         a_dict:\n",
-       " 15             a: 1\n",
-       " 16             b: 2\n",
-       " 17         a_list: [1, 2]\n",
-       " 18         a_string: "asdasd"\n",
-       " 19 \n",
-       " 20 host2.cmh:\n",
-       " 21     hostname: 127.0.0.1\n",
-       " 22     port: 2202\n",
-       " 23     username: vagrant\n",
-       " 24     password: vagrant\n",
-       " 25     site: cmh\n",
-       " 26     role: host\n",
+       "  7     platform: linux\n",
+       "  8     groups:\n",
+       "  9         - cmh\n",
+       " 10     data:\n",
+       " 11         site: cmh\n",
+       " 12         role: host\n",
+       " 13         type: host\n",
+       " 14         nested_data:\n",
+       " 15             a_dict:\n",
+       " 16                 a: 1\n",
+       " 17                 b: 2\n",
+       " 18             a_list: [1, 2]\n",
+       " 19             a_string: "asdasd"\n",
+       " 20 \n",
+       " 21 host2.cmh:\n",
+       " 22     hostname: 127.0.0.1\n",
+       " 23     port: 2202\n",
+       " 24     username: vagrant\n",
+       " 25     password: vagrant\n",
+       " 26     platform: linux\n",
        " 27     groups:\n",
        " 28         - cmh\n",
-       " 29     platform: linux\n",
-       " 30     type: host\n",
-       " 31     nested_data:\n",
-       " 32         a_dict:\n",
-       " 33             b: 2\n",
-       " 34             c: 3\n",
-       " 35         a_list: [1, 2]\n",
-       " 36         a_string: "qwe"\n",
-       " 37 \n",
-       " 38 spine00.cmh:\n",
-       " 39     hostname: 127.0.0.1\n",
-       " 40     username: vagrant\n",
-       " 41     password: vagrant\n",
-       " 42     port: 12444\n",
-       " 43     site: cmh\n",
-       " 44     role: spine\n",
-       " 45     groups:\n",
-       " 46         - cmh\n",
-       " 47     platform: eos\n",
-       " 48     type: network_device\n",
-       " 49 \n",
-       " 50 spine01.cmh:\n",
-       " 51     hostname: 127.0.0.1\n",
-       " 52     username: vagrant\n",
-       " 53     password: ""\n",
-       " 54     port: 12204\n",
-       " 55     site: cmh\n",
-       " 56     role: spine\n",
-       " 57     groups:\n",
-       " 58         - cmh\n",
-       " 59     platform: junos\n",
-       " 60     type: network_device\n",
-       " 61 \n",
-       " 62 leaf00.cmh:\n",
-       " 63     hostname: 127.0.0.1\n",
-       " 64     username: vagrant\n",
-       " 65     password: vagrant\n",
-       " 66     port: 12443\n",
-       " 67     site: cmh\n",
-       " 68     role: leaf\n",
-       " 69     groups:\n",
-       " 70         - cmh\n",
+       " 29     data:\n",
+       " 30         site: cmh\n",
+       " 31         role: host\n",
+       " 32         type: host\n",
+       " 33         nested_data:\n",
+       " 34             a_dict:\n",
+       " 35                 b: 2\n",
+       " 36                 c: 3\n",
+       " 37             a_list: [1, 2]\n",
+       " 38             a_string: "qwe"\n",
+       " 39 \n",
+       " 40 spine00.cmh:\n",
+       " 41     hostname: 127.0.0.1\n",
+       " 42     username: vagrant\n",
+       " 43     password: vagrant\n",
+       " 44     port: 12444\n",
+       " 45     platform: eos\n",
+       " 46     groups:\n",
+       " 47         - cmh\n",
+       " 48     data:\n",
+       " 49         site: cmh\n",
+       " 50         role: spine\n",
+       " 51         type: network_device\n",
+       " 52 \n",
+       " 53 spine01.cmh:\n",
+       " 54     hostname: 127.0.0.1\n",
+       " 55     username: vagrant\n",
+       " 56     password: ""\n",
+       " 57     platform: junos\n",
+       " 58     port: 12204\n",
+       " 59     groups:\n",
+       " 60         - cmh\n",
+       " 61     data:\n",
+       " 62         site: cmh\n",
+       " 63         role: spine\n",
+       " 64         type: network_device\n",
+       " 65 \n",
+       " 66 leaf00.cmh:\n",
+       " 67     hostname: 127.0.0.1\n",
+       " 68     username: vagrant\n",
+       " 69     password: vagrant\n",
+       " 70     port: 12443\n",
        " 71     platform: eos\n",
-       " 72     type: network_device\n",
-       " 73     asn: 65100\n",
-       " 74 \n",
-       " 75 leaf01.cmh:\n",
-       " 76     hostname: 127.0.0.1\n",
-       " 77     username: vagrant\n",
-       " 78     password: ""\n",
-       " 79     port: 12203\n",
-       " 80     site: cmh\n",
-       " 81     role: leaf\n",
-       " 82     groups:\n",
-       " 83         - cmh\n",
-       " 84     platform: junos\n",
-       " 85     type: network_device\n",
-       " 86     asn: 65101\n",
-       " 87 \n",
-       " 88 host1.bma:\n",
-       " 89     site: bma\n",
-       " 90     role: host\n",
-       " 91     groups:\n",
-       " 92         - bma\n",
-       " 93     platform: linux\n",
-       " 94     type: host\n",
-       " 95 \n",
-       " 96 host2.bma:\n",
-       " 97     site: bma\n",
-       " 98     role: host\n",
-       " 99     groups:\n",
-       "100         - bma\n",
-       "101     platform: linux\n",
-       "102     type: host\n",
-       "103 \n",
-       "104 spine00.bma:\n",
-       "105     hostname: 127.0.0.1\n",
-       "106     username: vagrant\n",
-       "107     password: vagrant\n",
-       "108     port: 12444\n",
-       "109     site: bma\n",
-       "110     role: spine\n",
-       "111     groups:\n",
-       "112         - bma\n",
-       "113     platform: eos\n",
-       "114     type: network_device\n",
-       "115 \n",
-       "116 spine01.bma:\n",
-       "117     hostname: 127.0.0.1\n",
-       "118     username: vagrant\n",
-       "119     password: ""\n",
-       "120     port: 12204\n",
-       "121     site: bma\n",
-       "122     role: spine\n",
-       "123     groups:\n",
-       "124         - bma\n",
-       "125     platform: junos\n",
-       "126     type: network_device\n",
-       "127 \n",
-       "128 leaf00.bma:\n",
-       "129     hostname: 127.0.0.1\n",
-       "130     username: vagrant\n",
-       "131     password: vagrant\n",
-       "132     port: 12443\n",
-       "133     site: bma\n",
-       "134     role: leaf\n",
-       "135     groups:\n",
-       "136         - bma\n",
-       "137     platform: eos\n",
-       "138     type: network_device\n",
-       "139 \n",
-       "140 leaf01.bma:\n",
-       "141     hostname: 127.0.0.1\n",
-       "142     username: vagrant\n",
-       "143     password: wrong_password\n",
-       "144     port: 12203\n",
-       "145     site: bma\n",
-       "146     role: leaf\n",
-       "147     groups:\n",
-       "148         - bma\n",
-       "149     platform: junos\n",
-       "150     type: network_device\n",
+       " 72     groups:\n",
+       " 73         - cmh\n",
+       " 74     data:\n",
+       " 75         site: cmh\n",
+       " 76         role: leaf\n",
+       " 77         type: network_device\n",
+       " 78         asn: 65100\n",
+       " 79 \n",
+       " 80 leaf01.cmh:\n",
+       " 81     hostname: 127.0.0.1\n",
+       " 82     username: vagrant\n",
+       " 83     password: ""\n",
+       " 84     port: 12203\n",
+       " 85     platform: junos\n",
+       " 86     groups:\n",
+       " 87         - cmh\n",
+       " 88     data:\n",
+       " 89         site: cmh\n",
+       " 90         role: leaf\n",
+       " 91         type: network_device\n",
+       " 92         asn: 65101\n",
+       " 93 \n",
+       " 94 host1.bma:\n",
+       " 95     groups:\n",
+       " 96         - bma\n",
+       " 97     platform: linux\n",
+       " 98     data:\n",
+       " 99         site: bma\n",
+       "100         role: host\n",
+       "101         type: host\n",
+       "102 \n",
+       "103 host2.bma:\n",
+       "104     groups:\n",
+       "105         - bma\n",
+       "106     platform: linux\n",
+       "107     data:\n",
+       "108         site: bma\n",
+       "109         role: host\n",
+       "110         type: host\n",
+       "111 \n",
+       "112 spine00.bma:\n",
+       "113     hostname: 127.0.0.1\n",
+       "114     username: vagrant\n",
+       "115     password: vagrant\n",
+       "116     port: 12444\n",
+       "117     platform: eos\n",
+       "118     groups:\n",
+       "119         - bma\n",
+       "120     data:\n",
+       "121         site: bma\n",
+       "122         role: spine\n",
+       "123         type: network_device\n",
+       "124 \n",
+       "125 spine01.bma:\n",
+       "126     hostname: 127.0.0.1\n",
+       "127     username: vagrant\n",
+       "128     password: ""\n",
+       "129     port: 12204\n",
+       "130     platform: junos\n",
+       "131     groups:\n",
+       "132         - bma\n",
+       "133     data:\n",
+       "134         site: bma\n",
+       "135         role: spine\n",
+       "136         type: network_device\n",
+       "137 \n",
+       "138 leaf00.bma:\n",
+       "139     hostname: 127.0.0.1\n",
+       "140     username: vagrant\n",
+       "141     password: vagrant\n",
+       "142     port: 12443\n",
+       "143     platform: eos\n",
+       "144     groups:\n",
+       "145         - bma\n",
+       "146     data:\n",
+       "147         site: bma\n",
+       "148         role: leaf\n",
+       "149         type: network_device\n",
+       "150 \n",
+       "151 leaf01.bma:\n",
+       "152     hostname: 127.0.0.1\n",
+       "153     username: vagrant\n",
+       "154     password: wrong_password\n",
+       "155     port: 12203\n",
+       "156     platform: junos\n",
+       "157     groups:\n",
+       "158         - bma\n",
+       "159     data:\n",
+       "160         site: bma\n",
+       "161         role: leaf\n",
+       "162         type: network_device\n",
        "
\n", "\n" ], @@ -276,15 +288,128 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The hosts file is basically a map where the outermost key is the hostname and then any arbitrary `` pair you want inside. Some keys like hostname, username or password have special meaning, you can investigate the [hosts](../../ref/api/inventory.rst#nornir.core.inventory.Host) class for details on those. In addition, the `groups` key is a list of groups you can inherite data from. We will inspect soon how the inheritance model works.\n", - "\n", - "Now, let's look at the groups file:" + "The hosts file is basically a map where the outermost key is the name of the host and then a [InventoryElemant](../../ref/api/inventory.rst#nornir.core.inventory.InventoryElement) object. You can see the schema of the object by executing:" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"title\": \"InventoryElement\",\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"hostname\": {\n", + " \"title\": \"Hostname\",\n", + " \"required\": false,\n", + " \"type\": \"str\"\n", + " },\n", + " \"port\": {\n", + " \"title\": \"Port\",\n", + " \"required\": false,\n", + " \"type\": \"int\"\n", + " },\n", + " \"username\": {\n", + " \"title\": \"Username\",\n", + " \"required\": false,\n", + " \"type\": \"str\"\n", + " },\n", + " \"password\": {\n", + " \"title\": \"Password\",\n", + " \"required\": false,\n", + " \"type\": \"str\"\n", + " },\n", + " \"platform\": {\n", + " \"title\": \"Platform\",\n", + " \"required\": false,\n", + " \"type\": \"str\"\n", + " },\n", + " \"groups\": {\n", + " \"title\": \"Groups\",\n", + " \"required\": false,\n", + " \"default\": [],\n", + " \"type\": \"list\",\n", + " \"item_type\": \"str\"\n", + " },\n", + " \"data\": {\n", + " \"title\": \"Data\",\n", + " \"required\": false,\n", + " \"default\": {},\n", + " \"type\": \"mapping\",\n", + " \"item_type\": \"any\",\n", + " \"key_type\": \"str\"\n", + " },\n", + " \"connection_options\": {\n", + " \"title\": \"Connection_Options\",\n", + " \"required\": false,\n", + " \"default\": {},\n", + " \"type\": \"mapping\",\n", + " \"item_type\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"hostname\": {\n", + " \"title\": \"Hostname\",\n", + " \"required\": false,\n", + " \"type\": \"str\"\n", + " },\n", + " \"port\": {\n", + " \"title\": \"Port\",\n", + " \"required\": false,\n", + " \"type\": \"int\"\n", + " },\n", + " \"username\": {\n", + " \"title\": \"Username\",\n", + " \"required\": false,\n", + " \"type\": \"str\"\n", + " },\n", + " \"password\": {\n", + " \"title\": \"Password\",\n", + " \"required\": false,\n", + " \"type\": \"str\"\n", + " },\n", + " \"platform\": {\n", + " \"title\": \"Platform\",\n", + " \"required\": false,\n", + " \"type\": \"str\"\n", + " },\n", + " \"extras\": {\n", + " \"title\": \"Extras\",\n", + " \"required\": false,\n", + " \"type\": \"mapping\",\n", + " \"item_type\": \"any\",\n", + " \"key_type\": \"str\"\n", + " }\n", + " }\n", + " },\n", + " \"key_type\": \"str\"\n", + " }\n", + " }\n", + "}\n" + ] + } + ], + "source": [ + "from nornir.core.deserializer.inventory import InventoryElement\n", + "import json\n", + "print(json.dumps(InventoryElement.schema(), indent=4))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `groups_file` follows the same rules as the `hosts_file`." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, "outputs": [ { "data": { @@ -364,26 +489,26 @@ "\n", "\n", "
 1 ---\n",
-       " 2 defaults:\n",
-       " 3     domain: acme.local\n",
-       " 4 \n",
-       " 5 global:\n",
-       " 6     domain: global.local\n",
-       " 7     asn: 1\n",
-       " 8 \n",
-       " 9 eu:\n",
-       "10     asn: 65100\n",
-       "11 \n",
-       "12 bma:\n",
-       "13     groups:\n",
-       "14         - eu\n",
-       "15         - global\n",
-       "16 \n",
-       "17 cmh:\n",
-       "18     asn: 65000\n",
-       "19     vlans:\n",
-       "20       100: frontend\n",
-       "21       200: backend\n",
+       " 2 global:\n",
+       " 3     data:\n",
+       " 4         domain: global.local\n",
+       " 5         asn: 1\n",
+       " 6 \n",
+       " 7 eu:\n",
+       " 8     data:\n",
+       " 9         asn: 65100\n",
+       "10 \n",
+       "11 bma:\n",
+       "12     groups:\n",
+       "13         - eu\n",
+       "14         - global\n",
+       "15 \n",
+       "16 cmh:\n",
+       "17     data:\n",
+       "18         asn: 65000\n",
+       "19         vlans:\n",
+       "20           100: frontend\n",
+       "21           200: backend\n",
        "
\n", "\n" ], @@ -391,7 +516,7 @@ "" ] }, - "execution_count": 3, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -405,34 +530,138 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Pretty similar to the hosts file.\n", - "\n", - "### Accessing the inventory\n", - "\n", - "You can access the [inventory](../../ref/api/inventory.rst#nornir.core.inventory.Inventory) with the `inventory` attribute:" + "Finally, the defaults file has the same schema as the `InventoryElement` we described before but without outer keys to denote individual elements. We will see how the data in the groups and defaults file is used later on in this tutorial." ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { "data": { + "text/html": [ + "\n", + "
1 ---\n",
+       "2 data:\n",
+       "3     domain: acme.local\n",
+       "
\n", + "\n" + ], "text/plain": [ - "" + "" ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], + "source": [ + "# defaults file\n", + "%highlight_file inventory/defaults.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Accessing the inventory\n", + "\n", + "You can access the [inventory](../../ref/api/inventory.rst#nornir.core.inventory.Inventory) with the `inventory` attribute:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'host1.cmh': Host: host1.cmh, 'host2.cmh': Host: host2.cmh, 'spine00.cmh': Host: spine00.cmh, 'spine01.cmh': Host: spine01.cmh, 'leaf00.cmh': Host: leaf00.cmh, 'leaf01.cmh': Host: leaf01.cmh, 'host1.bma': Host: host1.bma, 'host2.bma': Host: host2.bma, 'spine00.bma': Host: spine00.bma, 'spine01.bma': Host: spine01.bma, 'leaf00.bma': Host: leaf00.bma, 'leaf01.bma': Host: leaf01.bma}\n" + ] + } + ], "source": [ "from nornir import InitNornir\n", "nr = InitNornir(config_file=\"config.yaml\")\n", "\n", - "nr.inventory" + "print(nr.inventory.hosts)" ] }, { @@ -444,7 +673,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -464,7 +693,7 @@ " 'leaf01.bma': Host: leaf01.bma}" ] }, - "execution_count": 5, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -475,7 +704,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -487,7 +716,7 @@ " 'cmh': Group: cmh}" ] }, - "execution_count": 6, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -498,7 +727,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -507,7 +736,7 @@ "Host: leaf01.bma" ] }, - "execution_count": 7, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -525,16 +754,16 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "dict_keys(['name', 'groups', 'hostname', 'username', 'password', 'port', 'site', 'role', 'platform', 'type', 'asn', 'domain'])" + "dict_keys(['site', 'role', 'type', 'asn', 'domain'])" ] }, - "execution_count": 8, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -546,7 +775,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 11, "metadata": { "scrolled": true }, @@ -557,7 +786,7 @@ "'bma'" ] }, - "execution_count": 9, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -577,7 +806,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -658,26 +887,26 @@ "\n", "\n", "
 1 ---\n",
-       " 2 defaults:\n",
-       " 3     domain: acme.local\n",
-       " 4 \n",
-       " 5 global:\n",
-       " 6     domain: global.local\n",
-       " 7     asn: 1\n",
-       " 8 \n",
-       " 9 eu:\n",
-       "10     asn: 65100\n",
-       "11 \n",
-       "12 bma:\n",
-       "13     groups:\n",
-       "14         - eu\n",
-       "15         - global\n",
-       "16 \n",
-       "17 cmh:\n",
-       "18     asn: 65000\n",
-       "19     vlans:\n",
-       "20       100: frontend\n",
-       "21       200: backend\n",
+       " 2 global:\n",
+       " 3     data:\n",
+       " 4         domain: global.local\n",
+       " 5         asn: 1\n",
+       " 6 \n",
+       " 7 eu:\n",
+       " 8     data:\n",
+       " 9         asn: 65100\n",
+       "10 \n",
+       "11 bma:\n",
+       "12     groups:\n",
+       "13         - eu\n",
+       "14         - global\n",
+       "15 \n",
+       "16 cmh:\n",
+       "17     data:\n",
+       "18         asn: 65000\n",
+       "19         vlans:\n",
+       "20           100: frontend\n",
+       "21           200: backend\n",
        "
\n", "\n" ], @@ -685,7 +914,7 @@ "" ] }, - "execution_count": 10, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -706,7 +935,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -715,7 +944,7 @@ "'global.local'" ] }, - "execution_count": 11, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -727,7 +956,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -736,7 +965,7 @@ "65100" ] }, - "execution_count": 12, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -749,12 +978,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The group `defaults` is special. This group contains data that will be returned if neither the host nor the parents have a specific value for it." + "Values in `defaults` will be returned if neither the host nor the parents have a specific value for it." ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -763,7 +992,7 @@ "'acme.local'" ] }, - "execution_count": 13, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -782,7 +1011,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -809,7 +1038,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -840,7 +1069,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -849,7 +1078,7 @@ "dict_keys(['host1.cmh', 'host2.cmh', 'spine00.cmh', 'spine01.cmh', 'leaf00.cmh', 'leaf01.cmh'])" ] }, - "execution_count": 16, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -867,7 +1096,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -876,7 +1105,7 @@ "dict_keys(['spine00.cmh', 'spine01.cmh'])" ] }, - "execution_count": 17, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -894,7 +1123,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -903,7 +1132,7 @@ "dict_keys(['spine00.cmh', 'spine01.cmh'])" ] }, - "execution_count": 18, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -921,7 +1150,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -930,7 +1159,7 @@ "dict_keys(['spine00.cmh', 'spine01.cmh'])" ] }, - "execution_count": 19, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -942,7 +1171,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -951,7 +1180,7 @@ "dict_keys(['leaf00.cmh', 'leaf01.cmh'])" ] }, - "execution_count": 20, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -969,27 +1198,27 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 23, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'host1.bma': Host: host1.bma,\n", - " 'host2.bma': Host: host2.bma,\n", - " 'spine00.bma': Host: spine00.bma,\n", - " 'spine01.bma': Host: spine01.bma,\n", - " 'leaf00.bma': Host: leaf00.bma,\n", - " 'leaf01.bma': Host: leaf01.bma}" + "{Host: host1.bma,\n", + " Host: host2.bma,\n", + " Host: leaf00.bma,\n", + " Host: leaf01.bma,\n", + " Host: spine00.bma,\n", + " Host: spine01.bma}" ] }, - "execution_count": 21, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "nr.inventory.groups[\"eu\"].children()" + "nr.inventory.children_of_group(\"eu\")" ] }, { @@ -1010,7 +1239,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 24, "metadata": {}, "outputs": [ { @@ -1019,7 +1248,7 @@ "dict_keys(['spine00.cmh', 'spine01.cmh', 'spine00.bma', 'spine01.bma'])" ] }, - "execution_count": 22, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -1033,7 +1262,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 25, "metadata": {}, "outputs": [ { @@ -1042,7 +1271,7 @@ "dict_keys(['host1.cmh', 'host2.cmh', 'host1.bma', 'host2.bma'])" ] }, - "execution_count": 23, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -1063,7 +1292,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -1073,7 +1302,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -1092,7 +1321,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -1111,7 +1340,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 29, "metadata": {}, "outputs": [ { @@ -1130,7 +1359,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -1156,7 +1385,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 31, "metadata": {}, "outputs": [ { @@ -1174,7 +1403,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 32, "metadata": {}, "outputs": [ { @@ -1192,7 +1421,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 33, "metadata": {}, "outputs": [ { @@ -1233,7 +1462,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.7.0" } }, "nbformat": 4, diff --git a/docs/howto/inventory/defaults.yaml b/docs/tutorials/intro/inventory/defaults.yaml similarity index 67% rename from docs/howto/inventory/defaults.yaml rename to docs/tutorials/intro/inventory/defaults.yaml index ddbc8dff..38d3b12e 100644 --- a/docs/howto/inventory/defaults.yaml +++ b/docs/tutorials/intro/inventory/defaults.yaml @@ -1,4 +1,3 @@ --- -username: admin data: domain: acme.local diff --git a/docs/tutorials/intro/inventory/groups.yaml b/docs/tutorials/intro/inventory/groups.yaml index 451d6bae..0a9fc619 100644 --- a/docs/tutorials/intro/inventory/groups.yaml +++ b/docs/tutorials/intro/inventory/groups.yaml @@ -1,13 +1,12 @@ --- -defaults: - domain: acme.local - global: - domain: global.local - asn: 1 + data: + domain: global.local + asn: 1 eu: - asn: 65100 + data: + asn: 65100 bma: groups: @@ -15,7 +14,8 @@ bma: - global cmh: - asn: 65000 - vlans: - 100: frontend - 200: backend + data: + asn: 65000 + vlans: + 100: frontend + 200: backend diff --git a/docs/tutorials/intro/inventory/hosts.yaml b/docs/tutorials/intro/inventory/hosts.yaml index 748b0d70..c8081951 100644 --- a/docs/tutorials/intro/inventory/hosts.yaml +++ b/docs/tutorials/intro/inventory/hosts.yaml @@ -4,147 +4,159 @@ host1.cmh: port: 2201 username: vagrant password: vagrant - site: cmh - role: host + platform: linux groups: - cmh - platform: linux - type: host - nested_data: - a_dict: - a: 1 - b: 2 - a_list: [1, 2] - a_string: "asdasd" + data: + site: cmh + role: host + type: host + nested_data: + a_dict: + a: 1 + b: 2 + a_list: [1, 2] + a_string: "asdasd" host2.cmh: hostname: 127.0.0.1 port: 2202 username: vagrant password: vagrant - site: cmh - role: host + platform: linux groups: - cmh - platform: linux - type: host - nested_data: - a_dict: - b: 2 - c: 3 - a_list: [1, 2] - a_string: "qwe" + data: + site: cmh + role: host + type: host + nested_data: + a_dict: + b: 2 + c: 3 + a_list: [1, 2] + a_string: "qwe" spine00.cmh: hostname: 127.0.0.1 username: vagrant password: vagrant port: 12444 - site: cmh - role: spine + platform: eos groups: - cmh - platform: eos - type: network_device + data: + site: cmh + role: spine + type: network_device spine01.cmh: hostname: 127.0.0.1 username: vagrant password: "" + platform: junos port: 12204 - site: cmh - role: spine groups: - cmh - platform: junos - type: network_device + data: + site: cmh + role: spine + type: network_device leaf00.cmh: hostname: 127.0.0.1 username: vagrant password: vagrant port: 12443 - site: cmh - role: leaf + platform: eos groups: - cmh - platform: eos - type: network_device - asn: 65100 + data: + site: cmh + role: leaf + type: network_device + asn: 65100 leaf01.cmh: hostname: 127.0.0.1 username: vagrant password: "" port: 12203 - site: cmh - role: leaf + platform: junos groups: - cmh - platform: junos - type: network_device - asn: 65101 + data: + site: cmh + role: leaf + type: network_device + asn: 65101 host1.bma: - site: bma - role: host groups: - bma platform: linux - type: host + data: + site: bma + role: host + type: host host2.bma: - site: bma - role: host groups: - bma platform: linux - type: host + data: + site: bma + role: host + type: host spine00.bma: hostname: 127.0.0.1 username: vagrant password: vagrant port: 12444 - site: bma - role: spine + platform: eos groups: - bma - platform: eos - type: network_device + data: + site: bma + role: spine + type: network_device spine01.bma: hostname: 127.0.0.1 username: vagrant password: "" port: 12204 - site: bma - role: spine + platform: junos groups: - bma - platform: junos - type: network_device + data: + site: bma + role: spine + type: network_device leaf00.bma: hostname: 127.0.0.1 username: vagrant password: vagrant port: 12443 - site: bma - role: leaf + platform: eos groups: - bma - platform: eos - type: network_device + data: + site: bma + role: leaf + type: network_device leaf01.bma: hostname: 127.0.0.1 username: vagrant password: wrong_password port: 12203 - site: bma - role: leaf + platform: junos groups: - bma - platform: junos - type: network_device + data: + site: bma + role: leaf + type: network_device diff --git a/docs/tutorials/intro/task_results.ipynb b/docs/tutorials/intro/task_results.ipynb index 6d4dcac9..e9dc529f 100644 --- a/docs/tutorials/intro/task_results.ipynb +++ b/docs/tutorials/intro/task_results.ipynb @@ -493,7 +493,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.7.0" } }, "nbformat": 4, diff --git a/docs/tutorials/intro/templates/eos/base.j2 b/docs/tutorials/intro/templates/eos/base.j2 index 692a00cd..9675effe 100644 --- a/docs/tutorials/intro/templates/eos/base.j2 +++ b/docs/tutorials/intro/templates/eos/base.j2 @@ -1,2 +1,2 @@ hostname {{ host }} -ip domain-name {{ site }}.{{ domain }} +ip domain-name {{ host.site }}.{{ host.domain }} diff --git a/docs/tutorials/intro/templates/junos/base.j2 b/docs/tutorials/intro/templates/junos/base.j2 index 775270c8..290fe320 100644 --- a/docs/tutorials/intro/templates/junos/base.j2 +++ b/docs/tutorials/intro/templates/junos/base.j2 @@ -1,4 +1,4 @@ system { host-name {{ host }}; - domain-name {{ site }}.{{ domain }}; + domain-name {{ host.site }}.{{ host.domain }}; } diff --git a/setup.py b/setup.py index 0055495c..9fdf21f0 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ __author__ = "dbarrosop@dravetech.com" __license__ = "Apache License, version 2" -__version__ = "2.0.0" +__version__ = "2.0.0b1" setup( name="nornir", diff --git a/tox.ini b/tox.ini index f5a712d2..72cdc8c5 100644 --- a/tox.ini +++ b/tox.ini @@ -49,3 +49,5 @@ deps = basepython = python3.6 commands = pytest --nbval docs/howto + pytest --nbval docs/tutorials/intro/initializing_nornir.ipynb + pytest --nbval docs/tutorials/intro/inventory.ipynb From 4c25c9f5637ac55c2662ee6d248440a351fc1196 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sat, 27 Oct 2018 19:25:51 +0200 Subject: [PATCH 091/109] fix sphinx --- docs/tutorials/intro/inventory.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/intro/inventory.ipynb b/docs/tutorials/intro/inventory.ipynb index 7f46b1c3..87e69198 100644 --- a/docs/tutorials/intro/inventory.ipynb +++ b/docs/tutorials/intro/inventory.ipynb @@ -288,7 +288,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The hosts file is basically a map where the outermost key is the name of the host and then a [InventoryElemant](../../ref/api/inventory.rst#nornir.core.inventory.InventoryElement) object. You can see the schema of the object by executing:" + "The hosts file is basically a map where the outermost key is the name of the host and then an `InventoryElemant` object. You can see the schema of the object by executing:" ] }, { From c916502f241ed67751a4041cb2856b4dbd65b3c0 Mon Sep 17 00:00:00 2001 From: Walid Amer Date: Tue, 30 Oct 2018 21:36:34 +0300 Subject: [PATCH 092/109] Update the netmiko map with nxos_ssh --- nornir/plugins/connections/netmiko.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nornir/plugins/connections/netmiko.py b/nornir/plugins/connections/netmiko.py index 58801ffe..eb1ee5f9 100644 --- a/nornir/plugins/connections/netmiko.py +++ b/nornir/plugins/connections/netmiko.py @@ -8,6 +8,7 @@ napalm_to_netmiko_map = { "ios": "cisco_ios", "nxos": "cisco_nxos", + "nxos_ssh": "cisco_nxos", "eos": "arista_eos", "junos": "juniper_junos", "iosxr": "cisco_xr", From 1cdd805cdba71c7523dce9e45b6a89f1947c3797 Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Wed, 31 Oct 2018 10:45:19 -0700 Subject: [PATCH 093/109] Add enable argument to Netmiko send_command to make enable support easier --- nornir/plugins/tasks/networking/netmiko_send_command.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nornir/plugins/tasks/networking/netmiko_send_command.py b/nornir/plugins/tasks/networking/netmiko_send_command.py index 5165a884..251e5255 100644 --- a/nornir/plugins/tasks/networking/netmiko_send_command.py +++ b/nornir/plugins/tasks/networking/netmiko_send_command.py @@ -4,7 +4,11 @@ def netmiko_send_command( - task: Task, command_string: str, use_timing: bool = False, **kwargs: Any + task: Task, + command_string: str, + use_timing: bool = False, + enable: bool = False, + **kwargs: Any ) -> Result: """ Execute Netmiko send_command method (or send_command_timing) @@ -19,6 +23,8 @@ def netmiko_send_command( * result (``dict``): dictionary with the result of the show command. """ net_connect = task.host.get_connection("netmiko", task.nornir.config) + if enable: + net_connect.enable() if use_timing: result = net_connect.send_command_timing(command_string, **kwargs) else: From be11ea23a9843c23d1a55205e761175d002974c3 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Fri, 9 Nov 2018 14:05:07 +0100 Subject: [PATCH 094/109] fixes #270 --- nornir/core/filter.py | 5 ++++- tests/core/test_filter.py | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/nornir/core/filter.py b/nornir/core/filter.py index f71f35df..43fc39c2 100644 --- a/nornir/core/filter.py +++ b/nornir/core/filter.py @@ -47,7 +47,10 @@ def __repr__(self): @staticmethod def _verify_rules(data, rule, value): if len(rule) > 1: - return F._verify_rules(data.get(rule[0], {}), rule[1:], value) + try: + return F._verify_rules(data.get(rule[0], {}), rule[1:], value) + except AttributeError: + return False elif len(rule) == 1: operator = "__{}__".format(rule[0]) diff --git a/tests/core/test_filter.py b/tests/core/test_filter.py index a3524bc0..946a697a 100644 --- a/tests/core/test_filter.py +++ b/tests/core/test_filter.py @@ -130,3 +130,9 @@ def test_filtering_list_all(self, nornir): filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) assert filtered == ["dev1.group_1"] + + def test_filter_wrong_attribute_for_type(self, nornir): + f = F(port__startswith="a") + filtered = sorted(list((nornir.inventory.filter(f).hosts.keys()))) + + assert filtered == [] From 0864833b67e1bd953f4808a7970051994987240b Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Fri, 9 Nov 2018 10:01:50 -0800 Subject: [PATCH 095/109] Update docstring --- nornir/plugins/tasks/networking/netmiko_send_command.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nornir/plugins/tasks/networking/netmiko_send_command.py b/nornir/plugins/tasks/networking/netmiko_send_command.py index 251e5255..d533a5b0 100644 --- a/nornir/plugins/tasks/networking/netmiko_send_command.py +++ b/nornir/plugins/tasks/networking/netmiko_send_command.py @@ -16,6 +16,7 @@ def netmiko_send_command( Arguments: command_string: Command to execute on the remote network device. use_timing: Set to True to switch to send_command_timing method. + enable: Set to True to force Netmiko .enable() call. kwargs: Additional arguments to pass to send_command method. Returns: From f55b9d54f32241c2b4d30f331a1b39a35054ae68 Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Fri, 9 Nov 2018 10:07:30 -0800 Subject: [PATCH 096/109] Updating docstrings to fix errors. --- nornir/plugins/tasks/networking/netmiko_send_command.py | 2 +- nornir/plugins/tasks/networking/netmiko_send_config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nornir/plugins/tasks/networking/netmiko_send_command.py b/nornir/plugins/tasks/networking/netmiko_send_command.py index d533a5b0..2b281b77 100644 --- a/nornir/plugins/tasks/networking/netmiko_send_command.py +++ b/nornir/plugins/tasks/networking/netmiko_send_command.py @@ -21,7 +21,7 @@ def netmiko_send_command( Returns: Result object with the following attributes set: - * result (``dict``): dictionary with the result of the show command. + * result: Result of the show command (generally a string, but depends on use of TextFSM). """ net_connect = task.host.get_connection("netmiko", task.nornir.config) if enable: diff --git a/nornir/plugins/tasks/networking/netmiko_send_config.py b/nornir/plugins/tasks/networking/netmiko_send_config.py index 3876e64e..df95c5b8 100644 --- a/nornir/plugins/tasks/networking/netmiko_send_config.py +++ b/nornir/plugins/tasks/networking/netmiko_send_config.py @@ -19,7 +19,7 @@ def netmiko_send_config( Returns: Result object with the following attributes set: - * result (``dict``): dictionary showing the CLI from the configuration changes + * result (``str``): string showing the CLI from the configuration changes. """ net_connect = task.host.get_connection("netmiko", task.nornir.config) net_connect.enable() From 409e7cbf62f190f69f2a1f5a7f808fe326177159 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 11 Nov 2018 17:08:35 +0100 Subject: [PATCH 097/109] Fixed an issue deserializing ansible inventory --- nornir/core/deserializer/inventory.py | 1 + nornir/plugins/inventory/ansible.py | 75 ++++++++++++------- .../ansible/ini/expected/defaults.yaml | 9 +++ .../ansible/ini/expected/groups.yaml | 43 ++++++++--- .../inventory/ansible/ini/expected/hosts.yaml | 43 +++++++++-- .../ansible/yaml/expected/defaults.yaml | 9 +++ .../ansible/yaml/expected/groups.yaml | 43 ++++++++--- .../ansible/yaml/expected/hosts.yaml | 43 +++++++++-- .../ansible/yaml2/expected/defaults.yaml | 7 ++ .../ansible/yaml2/expected/groups.yaml | 2 +- .../ansible/yaml2/expected/hosts.yaml | 23 +++++- .../ansible/yaml3/expected/defaults.yaml | 12 +++ .../ansible/yaml3/expected/groups.yaml | 67 +++++++++++++++++ .../ansible/yaml3/expected/hosts.yaml | 61 +++++++++++++++ .../inventory/ansible/yaml3/source/hosts | 45 +++++++++++ tests/plugins/inventory/test_ansible.py | 32 +++++--- 16 files changed, 444 insertions(+), 71 deletions(-) create mode 100644 tests/plugins/inventory/ansible/ini/expected/defaults.yaml create mode 100644 tests/plugins/inventory/ansible/yaml/expected/defaults.yaml create mode 100644 tests/plugins/inventory/ansible/yaml2/expected/defaults.yaml create mode 100644 tests/plugins/inventory/ansible/yaml3/expected/defaults.yaml create mode 100644 tests/plugins/inventory/ansible/yaml3/expected/groups.yaml create mode 100644 tests/plugins/inventory/ansible/yaml3/expected/hosts.yaml create mode 100644 tests/plugins/inventory/ansible/yaml3/source/hosts diff --git a/nornir/core/deserializer/inventory.py b/nornir/core/deserializer/inventory.py index a9ebccad..3eec7cde 100644 --- a/nornir/core/deserializer/inventory.py +++ b/nornir/core/deserializer/inventory.py @@ -8,6 +8,7 @@ VarsDict = Dict[str, Any] HostsDict = Dict[str, VarsDict] GroupsDict = Dict[str, VarsDict] +DefaultsDict = VarsDict class BaseAttributes(BaseModel): diff --git a/nornir/plugins/inventory/ansible.py b/nornir/plugins/inventory/ansible.py index b0066b03..fdcc1560 100644 --- a/nornir/plugins/inventory/ansible.py +++ b/nornir/plugins/inventory/ansible.py @@ -3,23 +3,24 @@ import os from collections import defaultdict from pathlib import Path -from typing import Dict, Any, Tuple, Optional, cast, Union, MutableMapping, DefaultDict - -import ruamel.yaml - +from typing import Any, DefaultDict, Dict, MutableMapping, Optional, Tuple, Union, cast from mypy_extensions import TypedDict -from ruamel.yaml.scanner import ScannerError -from ruamel.yaml.composer import ComposerError - from nornir.core.deserializer.inventory import ( - Inventory, - VarsDict, + DefaultsDict, GroupsDict, HostsDict, + Inventory, + InventoryElement, + VarsDict, ) +import ruamel.yaml +from ruamel.yaml.composer import ComposerError +from ruamel.yaml.scanner import ScannerError + + VARS_FILENAME_EXTENSIONS = ["", ".yml", ".yaml"] @@ -45,6 +46,7 @@ def __init__(self, hostsfile: str) -> None: self.path = os.path.dirname(hostsfile) self.hosts: HostsDict = {} self.groups: GroupsDict = {} + self.defaults: DefaultsDict = {"data": {}} self.original_data: Optional[AnsibleGroupsDict] = None self.load_hosts_file() @@ -53,18 +55,20 @@ def parse_group( ) -> None: data = data or {} if group == "defaults": - self.groups[group] = {} group_file = "all" + dest_group = self.defaults else: self.add(group, self.groups) group_file = group + dest_group = self.groups[group] if parent and parent != "defaults": - self.groups[group]["groups"].append(parent) + dest_group["groups"].append(parent) - self.groups[group].update(data.get("vars", {})) - self.groups[group].update(self.read_vars_file(group_file, self.path, False)) - self.groups[group] = self.map_nornir_vars(self.groups[group]) + group_data = data.get("vars", {}) + vars_file_data = self.read_vars_file(group_file, self.path, False) or {} + self.normalize_data(dest_group, group_data, vars_file_data) + self.map_nornir_vars(dest_group) self.parse_hosts(data.get("hosts", {}), parent=group) @@ -86,9 +90,27 @@ def parse_hosts( self.add(host, self.hosts) if parent and parent != "defaults": self.hosts[host]["groups"].append(parent) - self.hosts[host].update(data) - self.hosts[host].update(self.read_vars_file(host, self.path, True)) - self.hosts[host] = self.map_nornir_vars(self.hosts[host]) + + vars_file_data = self.read_vars_file(host, self.path, True) + self.normalize_data(self.hosts[host], data, vars_file_data) + self.map_nornir_vars(self.hosts[host]) + + def normalize_data( + self, host: HostsDict, data: Dict[str, Any], vars_data: Dict[str, Any] + ) -> None: + reserved_fields = InventoryElement.__fields__.keys() + self.map_nornir_vars(data) + for k, v in data.items(): + if k in reserved_fields: + host[k] = v + else: + host["data"][k] = v + self.map_nornir_vars(vars_data) + for k, v in vars_data.items(): + if k in reserved_fields: + host[k] = v + else: + host["data"][k] = v def sort_groups(self) -> None: for host in self.hosts.values(): @@ -132,18 +154,14 @@ def map_nornir_vars(obj: VarsDict): "ansible_user": "username", "ansible_password": "password", } - result = {} - for k, v in obj.items(): - if k in mappings: - result[mappings[k]] = v - else: - result[k] = v - return result + for ansible_var, nornir_var in mappings.items(): + if ansible_var in obj: + obj[nornir_var] = obj.pop(ansible_var) @staticmethod def add(element: str, element_dict: Dict[str, VarsDict]) -> None: if element not in element_dict: - element_dict[element] = {"groups": []} + element_dict[element] = {"groups": [], "data": {}} def load_hosts_file(self) -> None: raise NotImplementedError @@ -226,7 +244,7 @@ def load_hosts_file(self) -> None: self.original_data = cast(AnsibleGroupsDict, YAML.load(f)) -def parse(hostsfile: str) -> Tuple[HostsDict, GroupsDict]: +def parse(hostsfile: str) -> Tuple[HostsDict, GroupsDict, DefaultsDict]: try: parser: AnsibleParser = INIParser(hostsfile) except cp.Error: @@ -240,13 +258,12 @@ def parse(hostsfile: str) -> Tuple[HostsDict, GroupsDict]: parser.parse() - return parser.hosts, parser.groups + return parser.hosts, parser.groups, parser.defaults class AnsibleInventory(Inventory): def __init__(self, hostsfile: str = "hosts", *args: Any, **kwargs: Any) -> None: - host_vars, group_vars = parse(hostsfile) - defaults = group_vars.pop("defaults") + host_vars, group_vars, defaults = parse(hostsfile) super().__init__( hosts=host_vars, groups=group_vars, defaults=defaults, *args, **kwargs ) diff --git a/tests/plugins/inventory/ansible/ini/expected/defaults.yaml b/tests/plugins/inventory/ansible/ini/expected/defaults.yaml new file mode 100644 index 00000000..3ea3be93 --- /dev/null +++ b/tests/plugins/inventory/ansible/ini/expected/defaults.yaml @@ -0,0 +1,9 @@ +connection_options: {} +data: + my_other_var: from_all + my_var: from_all +hostname: null +password: null +platform: null +port: null +username: null diff --git a/tests/plugins/inventory/ansible/ini/expected/groups.yaml b/tests/plugins/inventory/ansible/ini/expected/groups.yaml index c0d2d5e4..d39a17b2 100644 --- a/tests/plugins/inventory/ansible/ini/expected/groups.yaml +++ b/tests/plugins/inventory/ansible/ini/expected/groups.yaml @@ -1,19 +1,44 @@ dbservers: + connection_options: {} + data: + my_var: from_dbservers + my_yet_another_var: from_dbservers groups: - servers - my_var: from_dbservers - my_yet_another_var: from_dbservers -defaults: - my_other_var: from_all - my_var: from_all + hostname: null + password: null + platform: null + port: null + username: null frontend: + connection_options: {} + data: {} groups: [] + hostname: null + password: null + platform: null + port: null + username: null servers: - escape_pods: 2 + connection_options: {} + data: + escape_pods: 2 + halon_system_timeout: 30 + self_destruct_countdown: 60 + some_server: foo.southeast.example.com groups: [] - halon_system_timeout: 30 - self_destruct_countdown: 60 - some_server: foo.southeast.example.com + hostname: null + password: null + platform: null + port: null + username: null webservers: + connection_options: {} + data: {} groups: - servers + hostname: null + password: null + platform: null + port: null + username: null diff --git a/tests/plugins/inventory/ansible/ini/expected/hosts.yaml b/tests/plugins/inventory/ansible/ini/expected/hosts.yaml index 0fd8cdca..fc40190c 100644 --- a/tests/plugins/inventory/ansible/ini/expected/hosts.yaml +++ b/tests/plugins/inventory/ansible/ini/expected/hosts.yaml @@ -1,21 +1,54 @@ bar.example.com: + connection_options: {} + data: {} groups: - webservers + hostname: null + password: null + platform: null + port: null + username: null foo.example.com: + connection_options: {} + data: {} groups: - frontend - webservers + hostname: null + password: null + platform: null + port: null + username: null one.example.com: + connection_options: {} + data: + my_var: from_one.example.com groups: - dbservers - my_var: from_one.example.com + hostname: null + password: null + platform: null + port: null + username: null three.example.com: - hostname: 192.0.2.50 - port: 5555 + connection_options: {} + data: {} groups: - dbservers + hostname: 192.0.2.50 + password: null + platform: null + port: 5555 + username: null two.example.com: + connection_options: {} + data: + my_var: from_hostfile + whatever: asdasd groups: - dbservers - my_var: from_hostfile - whatever: asdasd + hostname: null + password: null + platform: null + port: null + username: null diff --git a/tests/plugins/inventory/ansible/yaml/expected/defaults.yaml b/tests/plugins/inventory/ansible/yaml/expected/defaults.yaml new file mode 100644 index 00000000..3ea3be93 --- /dev/null +++ b/tests/plugins/inventory/ansible/yaml/expected/defaults.yaml @@ -0,0 +1,9 @@ +connection_options: {} +data: + my_other_var: from_all + my_var: from_all +hostname: null +password: null +platform: null +port: null +username: null diff --git a/tests/plugins/inventory/ansible/yaml/expected/groups.yaml b/tests/plugins/inventory/ansible/yaml/expected/groups.yaml index c0d2d5e4..d39a17b2 100644 --- a/tests/plugins/inventory/ansible/yaml/expected/groups.yaml +++ b/tests/plugins/inventory/ansible/yaml/expected/groups.yaml @@ -1,19 +1,44 @@ dbservers: + connection_options: {} + data: + my_var: from_dbservers + my_yet_another_var: from_dbservers groups: - servers - my_var: from_dbservers - my_yet_another_var: from_dbservers -defaults: - my_other_var: from_all - my_var: from_all + hostname: null + password: null + platform: null + port: null + username: null frontend: + connection_options: {} + data: {} groups: [] + hostname: null + password: null + platform: null + port: null + username: null servers: - escape_pods: 2 + connection_options: {} + data: + escape_pods: 2 + halon_system_timeout: 30 + self_destruct_countdown: 60 + some_server: foo.southeast.example.com groups: [] - halon_system_timeout: 30 - self_destruct_countdown: 60 - some_server: foo.southeast.example.com + hostname: null + password: null + platform: null + port: null + username: null webservers: + connection_options: {} + data: {} groups: - servers + hostname: null + password: null + platform: null + port: null + username: null diff --git a/tests/plugins/inventory/ansible/yaml/expected/hosts.yaml b/tests/plugins/inventory/ansible/yaml/expected/hosts.yaml index 0fd8cdca..fc40190c 100644 --- a/tests/plugins/inventory/ansible/yaml/expected/hosts.yaml +++ b/tests/plugins/inventory/ansible/yaml/expected/hosts.yaml @@ -1,21 +1,54 @@ bar.example.com: + connection_options: {} + data: {} groups: - webservers + hostname: null + password: null + platform: null + port: null + username: null foo.example.com: + connection_options: {} + data: {} groups: - frontend - webservers + hostname: null + password: null + platform: null + port: null + username: null one.example.com: + connection_options: {} + data: + my_var: from_one.example.com groups: - dbservers - my_var: from_one.example.com + hostname: null + password: null + platform: null + port: null + username: null three.example.com: - hostname: 192.0.2.50 - port: 5555 + connection_options: {} + data: {} groups: - dbservers + hostname: 192.0.2.50 + password: null + platform: null + port: 5555 + username: null two.example.com: + connection_options: {} + data: + my_var: from_hostfile + whatever: asdasd groups: - dbservers - my_var: from_hostfile - whatever: asdasd + hostname: null + password: null + platform: null + port: null + username: null diff --git a/tests/plugins/inventory/ansible/yaml2/expected/defaults.yaml b/tests/plugins/inventory/ansible/yaml2/expected/defaults.yaml new file mode 100644 index 00000000..726111d9 --- /dev/null +++ b/tests/plugins/inventory/ansible/yaml2/expected/defaults.yaml @@ -0,0 +1,7 @@ +connection_options: {} +data: {} +hostname: null +password: null +platform: null +port: null +username: null diff --git a/tests/plugins/inventory/ansible/yaml2/expected/groups.yaml b/tests/plugins/inventory/ansible/yaml2/expected/groups.yaml index 914dee95..0967ef42 100644 --- a/tests/plugins/inventory/ansible/yaml2/expected/groups.yaml +++ b/tests/plugins/inventory/ansible/yaml2/expected/groups.yaml @@ -1 +1 @@ -defaults: {} +{} diff --git a/tests/plugins/inventory/ansible/yaml2/expected/hosts.yaml b/tests/plugins/inventory/ansible/yaml2/expected/hosts.yaml index 9e9b70c0..fa2a28a8 100644 --- a/tests/plugins/inventory/ansible/yaml2/expected/hosts.yaml +++ b/tests/plugins/inventory/ansible/yaml2/expected/hosts.yaml @@ -1,9 +1,28 @@ one.example.com: + connection_options: {} + data: {} groups: [] + hostname: null + password: null + platform: null + port: null + username: null three.example.com: + connection_options: {} + data: {} + groups: [] hostname: 192.0.2.50 + password: null + platform: null port: 5555 - groups: [] + username: null two.example.com: + connection_options: {} + data: + my_var: from_hostfile groups: [] - my_var: from_hostfile + hostname: null + password: null + platform: null + port: null + username: null diff --git a/tests/plugins/inventory/ansible/yaml3/expected/defaults.yaml b/tests/plugins/inventory/ansible/yaml3/expected/defaults.yaml new file mode 100644 index 00000000..c4c39304 --- /dev/null +++ b/tests/plugins/inventory/ansible/yaml3/expected/defaults.yaml @@ -0,0 +1,12 @@ +connection_options: {} +data: + ansible_connection: network_cli + ansible_network_os: ios + ansible_python_interpreter: python + ansible_ssh_common_args: -o ProxyCommand="ssh -W %h:%p -p 10000 guest@10.105.152.50" + env: staging +hostname: null +password: null +platform: null +port: null +username: null diff --git a/tests/plugins/inventory/ansible/yaml3/expected/groups.yaml b/tests/plugins/inventory/ansible/yaml3/expected/groups.yaml new file mode 100644 index 00000000..49e0ae73 --- /dev/null +++ b/tests/plugins/inventory/ansible/yaml3/expected/groups.yaml @@ -0,0 +1,67 @@ +access: + connection_options: {} + data: {} + groups: + - ios + hostname: null + password: null + platform: null + port: null + username: null +asa: + connection_options: {} + data: + ansible_become: yes + ansible_become_method: enable + ansible_network_os: asa + ansible_persistent_command_timeout: 30 + cisco_asa: true + network_os: asa + groups: + - virl + hostname: null + password: null + platform: null + port: null + username: null +dist: + connection_options: {} + data: {} + groups: + - ios + hostname: null + password: null + platform: null + port: null + username: null +ios: + connection_options: {} + data: + network_os: ios + nornir_nos: ios + groups: + - virl + hostname: null + password: null + platform: ios + port: null + username: null +routers: + connection_options: {} + data: {} + groups: + - ios + hostname: null + password: null + platform: null + port: null + username: null +virl: + connection_options: {} + data: {} + groups: [] + hostname: null + password: null + platform: null + port: null + username: null diff --git a/tests/plugins/inventory/ansible/yaml3/expected/hosts.yaml b/tests/plugins/inventory/ansible/yaml3/expected/hosts.yaml new file mode 100644 index 00000000..8a874e5d --- /dev/null +++ b/tests/plugins/inventory/ansible/yaml3/expected/hosts.yaml @@ -0,0 +1,61 @@ +access1: + connection_options: {} + data: {} + groups: + - access + hostname: 10.255.0.6 + password: null + platform: null + port: null + username: null +asa1: + connection_options: {} + data: + dot1x: true + groups: + - asa + hostname: 10.255.0.2 + password: null + platform: null + port: null + username: null +dist1: + connection_options: {} + data: {} + groups: + - dist + hostname: 10.255.0.5 + password: null + platform: null + port: null + username: null +dist2: + connection_options: {} + data: {} + groups: + - dist + hostname: 10.255.0.11 + password: null + platform: null + port: null + username: null +iosv-1: + connection_options: {} + data: {} + groups: + - routers + hostname: 10.255.0.12 + password: null + platform: null + port: null + username: null +iosv-2: + connection_options: {} + data: {} + groups: + - routers + hostname: 10.255.0.13 + password: null + platform: null + port: null + username: null diff --git a/tests/plugins/inventory/ansible/yaml3/source/hosts b/tests/plugins/inventory/ansible/yaml3/source/hosts new file mode 100644 index 00000000..177c4321 --- /dev/null +++ b/tests/plugins/inventory/ansible/yaml3/source/hosts @@ -0,0 +1,45 @@ +--- +all: + vars: + ansible_python_interpreter: python + ansible_connection: network_cli + ansible_network_os: ios + ansible_ssh_common_args: '-o ProxyCommand="ssh -W %h:%p -p 10000 guest@10.105.152.50"' # install ssh public key on proxy host !!! + env: staging + children: + virl: + children: + asa: + vars: + cisco_asa: True + network_os: asa + ansible_network_os: asa + ansible_become: yes + ansible_become_method: enable + ansible_persistent_command_timeout: 30 + hosts: + asa1: + ansible_host: 10.255.0.2 + dot1x: True + ios: + vars: + network_os: ios + nornir_nos: ios + platform: ios + children: + routers: + hosts: + iosv-1: + ansible_host: 10.255.0.12 + iosv-2: + ansible_host: 10.255.0.13 + access: + hosts: + access1: + ansible_host: 10.255.0.6 + dist: + hosts: + dist1: + ansible_host: 10.255.0.5 + dist2: + ansible_host: 10.255.0.11 diff --git a/tests/plugins/inventory/test_ansible.py b/tests/plugins/inventory/test_ansible.py index 800d5bad..199b7e34 100644 --- a/tests/plugins/inventory/test_ansible.py +++ b/tests/plugins/inventory/test_ansible.py @@ -11,39 +11,49 @@ BASE_PATH = os.path.join(os.path.dirname(__file__), "ansible") -def save(hosts, groups, hosts_file, groups_file): +def save(inv_serialized, hosts_file, groups_file, defaults_file): yml = ruamel.yaml.YAML(typ="safe") yml.default_flow_style = False with open(hosts_file, "w+") as f: - f.write(yml.dump(hosts)) + yml.dump(inv_serialized["hosts"], f) with open(groups_file, "w+") as f: - f.write(yml.dump(groups)) + yml.dump(inv_serialized["groups"], f) + with open(defaults_file, "w+") as f: + yml.dump(inv_serialized["defaults"], f) -def read(hosts_file, groups_file): +def read(hosts_file, groups_file, defaults_file): yml = ruamel.yaml.YAML(typ="safe") with open(hosts_file, "r") as f: hosts = yml.load(f) with open(groups_file, "r") as f: groups = yml.load(f) - return hosts, groups + with open(defaults_file, "r") as f: + defaults = yml.load(f) + return hosts, groups, defaults class Test(object): - @pytest.mark.parametrize("case", ["ini", "yaml", "yaml2"]) + @pytest.mark.parametrize("case", ["ini", "yaml", "yaml2", "yaml3"]) def test_inventory(self, case): base_path = os.path.join(BASE_PATH, case) hosts_file = os.path.join(base_path, "expected", "hosts.yaml") groups_file = os.path.join(base_path, "expected", "groups.yaml") + defaults_file = os.path.join(base_path, "expected", "defaults.yaml") - hosts, groups = ansible.parse( + inv = ansible.AnsibleInventory.deserialize( hostsfile=os.path.join(base_path, "source", "hosts") ) - # save(hosts, groups, hosts_file, groups_file) + inv_serialized = ansible.AnsibleInventory.serialize(inv).dict() - expected_hosts, expected_groups = read(hosts_file, groups_file) - assert hosts == expected_hosts - assert groups == expected_groups + # save(inv_serialized, hosts_file, groups_file, defaults_file) + + expected_hosts, expected_groups, expected_defaults = read( + hosts_file, groups_file, defaults_file + ) + assert inv_serialized["hosts"] == expected_hosts + assert inv_serialized["groups"] == expected_groups + assert inv_serialized["defaults"] == expected_defaults def test_parse_error(self): base_path = os.path.join(BASE_PATH, "parse_error") From 8b9641578d691a3208b9d1a38ff68e2c83d7cfd7 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 11 Nov 2018 17:17:59 +0100 Subject: [PATCH 098/109] Fixed an issue deserializing netbox inventory --- nornir/plugins/inventory/netbox.py | 2 +- tests/plugins/inventory/netbox/2.3.5/expected.json | 1 + .../inventory/netbox/2.3.5/expected_transform_function.json | 1 + tests/plugins/inventory/netbox/2.3.5/mocked/devices.json | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nornir/plugins/inventory/netbox.py b/nornir/plugins/inventory/netbox.py index 92488367..906198a4 100644 --- a/nornir/plugins/inventory/netbox.py +++ b/nornir/plugins/inventory/netbox.py @@ -42,7 +42,7 @@ def __init__( if flatten_custom_fields: for cf, value in d["custom_fields"].items(): - host[cf] = value + host["data"][cf] = value else: host["data"]["custom_fields"] = d["custom_fields"] diff --git a/tests/plugins/inventory/netbox/2.3.5/expected.json b/tests/plugins/inventory/netbox/2.3.5/expected.json index 52c78f0e..1af1e52f 100644 --- a/tests/plugins/inventory/netbox/2.3.5/expected.json +++ b/tests/plugins/inventory/netbox/2.3.5/expected.json @@ -41,6 +41,7 @@ "password": null, "platform": null, "data": { + "user_defined": 1, "serial": "", "vendor": "Cisco", "asset_tag": null, diff --git a/tests/plugins/inventory/netbox/2.3.5/expected_transform_function.json b/tests/plugins/inventory/netbox/2.3.5/expected_transform_function.json index df25fa9f..2889e0d1 100644 --- a/tests/plugins/inventory/netbox/2.3.5/expected_transform_function.json +++ b/tests/plugins/inventory/netbox/2.3.5/expected_transform_function.json @@ -43,6 +43,7 @@ "password": null, "platform": null, "data": { + "user_defined": 1, "serial": "", "vendor": "Cisco", "asset_tag": null, diff --git a/tests/plugins/inventory/netbox/2.3.5/mocked/devices.json b/tests/plugins/inventory/netbox/2.3.5/mocked/devices.json index cb707cd3..5b316a8a 100644 --- a/tests/plugins/inventory/netbox/2.3.5/mocked/devices.json +++ b/tests/plugins/inventory/netbox/2.3.5/mocked/devices.json @@ -185,7 +185,7 @@ "vc_position": null, "vc_priority": null, "comments": "", - "custom_fields": {}, + "custom_fields": {"user_defined": 1}, "created": "2018-07-12", "last_updated": "2018-07-12T11:53:54.866133Z" } From ce183653c05b8de551b68a16932142709b80ef5e Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 11 Nov 2018 17:26:06 +0100 Subject: [PATCH 099/109] Add query capabilities to netbox --- nornir/plugins/inventory/netbox.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/nornir/plugins/inventory/netbox.py b/nornir/plugins/inventory/netbox.py index 906198a4..1bcd76fe 100644 --- a/nornir/plugins/inventory/netbox.py +++ b/nornir/plugins/inventory/netbox.py @@ -13,8 +13,21 @@ def __init__( nb_token=None, use_slugs=True, flatten_custom_fields=True, - **kwargs + filter_parameters=None, + **kwargs, ) -> None: + """ + Netbox plugin + + Arguments: + nb_url: Netbox url, defaults to http://localhost:8080. + You can also use env variable NB_URL + nb_token: Netbokx token. You can also use env variable NB_TOKEN + use_slugs: Whether to use slugs or not + flatten_custom_fields: Whether to assign custom fields directly to the host or not + filter_parameters: Key-value pairs to filter down hosts + """ + filter_parameters = filter_parameters or {} nb_url = nb_url or os.environ.get("NB_URL", "http://localhost:8080") nb_token = nb_token or os.environ.get( @@ -23,7 +36,11 @@ def __init__( headers = {"Authorization": "Token {}".format(nb_token)} # Create dict of hosts using 'devices' from NetBox - r = requests.get("{}/api/dcim/devices/?limit=0".format(nb_url), headers=headers) + r = requests.get( + "{}/api/dcim/devices/?limit=0".format(nb_url), + headers=headers, + params=filter_parameters, + ) r.raise_for_status() nb_devices = r.json() From 5d76da5682b82ff66ae61f2f1d389e332bd2d697 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Tue, 13 Nov 2018 09:08:39 +0100 Subject: [PATCH 100/109] Release new beta --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9fdf21f0..5588f733 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ __author__ = "dbarrosop@dravetech.com" __license__ = "Apache License, version 2" -__version__ = "2.0.0b1" +__version__ = "2.0.0b2" setup( name="nornir", From 4165ff65b6db5c95edb20951354f700b022bd8fb Mon Sep 17 00:00:00 2001 From: Wim Van Deun <7521270+enzzzy@users.noreply.github.com> Date: Thu, 8 Nov 2018 23:35:25 +0100 Subject: [PATCH 101/109] initial plugin support for netmiko save_config() method --- nornir/plugins/tasks/networking/__init__.py | 2 ++ .../tasks/networking/netmiko_save_config.py | 27 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 nornir/plugins/tasks/networking/netmiko_save_config.py diff --git a/nornir/plugins/tasks/networking/__init__.py b/nornir/plugins/tasks/networking/__init__.py index 63cfd04b..16efede3 100644 --- a/nornir/plugins/tasks/networking/__init__.py +++ b/nornir/plugins/tasks/networking/__init__.py @@ -5,6 +5,7 @@ from .netmiko_file_transfer import netmiko_file_transfer from .netmiko_send_command import netmiko_send_command from .netmiko_send_config import netmiko_send_config +from .netmiko_save_config import netmiko_save_config from .tcp_ping import tcp_ping __all__ = ( @@ -15,5 +16,6 @@ "netmiko_file_transfer", "netmiko_send_command", "netmiko_send_config", + "netmiko_save_config", "tcp_ping", ) diff --git a/nornir/plugins/tasks/networking/netmiko_save_config.py b/nornir/plugins/tasks/networking/netmiko_save_config.py new file mode 100644 index 00000000..498ac259 --- /dev/null +++ b/nornir/plugins/tasks/networking/netmiko_save_config.py @@ -0,0 +1,27 @@ +from __future__ import unicode_literals + +from nornir.core.task import Result, Task + + +def netmiko_save_config( + task: Task, cmd: str = "", confirm: bool = False, confirm_response: str = "" +) -> Result: + """ + Execute Netmiko save_config method + Arguments: + cmd(str, optional): Command used to save the configuration. + confirm(bool, optional): Does device prompt for confirmation before executing save operation + confirm_repsonse(str, optional): Response send to device when it prompts for confirmation + + Returns: + :obj: `nornir.core.task.Result`: + * result (``dict``): dictionary showing the CLI from the save operation + """ + conn = task.host.get_connection("netmiko", task.nornir.config) + if cmd: + result = conn.save_config( + cmd=cmd, confirm=confirm, confirm_response=confirm_response + ) + else: + result = conn.save_config(confirm=confirm, confirm_response=confirm_response) + return Result(host=task.host, result=result, changed=True) From e53b660bc1f925f4b1ef9b677a145492039ebb18 Mon Sep 17 00:00:00 2001 From: Ravi B Date: Fri, 16 Nov 2018 11:42:56 +0100 Subject: [PATCH 102/109] Update inventory.ipynb fix typo on line 291 --- docs/tutorials/intro/inventory.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/intro/inventory.ipynb b/docs/tutorials/intro/inventory.ipynb index 87e69198..bcb5015a 100644 --- a/docs/tutorials/intro/inventory.ipynb +++ b/docs/tutorials/intro/inventory.ipynb @@ -288,7 +288,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The hosts file is basically a map where the outermost key is the name of the host and then an `InventoryElemant` object. You can see the schema of the object by executing:" + "The hosts file is basically a map where the outermost key is the name of the host and then an `InventoryElement` object. You can see the schema of the object by executing:" ] }, { From 81c7434cd30820b5ba4d430a9525b233d5ca5400 Mon Sep 17 00:00:00 2001 From: Kirk Byers Date: Mon, 19 Nov 2018 14:50:01 -0800 Subject: [PATCH 103/109] Minor corrections to some docstrings --- nornir/plugins/tasks/networking/netmiko_save_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nornir/plugins/tasks/networking/netmiko_save_config.py b/nornir/plugins/tasks/networking/netmiko_save_config.py index 498ac259..feec42cc 100644 --- a/nornir/plugins/tasks/networking/netmiko_save_config.py +++ b/nornir/plugins/tasks/networking/netmiko_save_config.py @@ -11,11 +11,11 @@ def netmiko_save_config( Arguments: cmd(str, optional): Command used to save the configuration. confirm(bool, optional): Does device prompt for confirmation before executing save operation - confirm_repsonse(str, optional): Response send to device when it prompts for confirmation + confirm_response(str, optional): Response send to device when it prompts for confirmation Returns: :obj: `nornir.core.task.Result`: - * result (``dict``): dictionary showing the CLI from the save operation + * result (``str``): String showing the CLI output from the save operation """ conn = task.host.get_connection("netmiko", task.nornir.config) if cmd: From a0ba0b84f87c43b61ec3e58016fc5c3e28e1dcd7 Mon Sep 17 00:00:00 2001 From: David Barroso Date: Tue, 20 Nov 2018 10:40:39 +0100 Subject: [PATCH 104/109] resolve falsey vars correctly --- nornir/core/inventory.py | 6 ++++-- tests/core/test_inventory.py | 1 + tests/inventory_data/groups.yaml | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 8e719b7b..56f4587d 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -153,9 +153,11 @@ def __getitem__(self, item): except KeyError: for g in self.groups.refs: - r = g.get(item) - if r: + try: + r = g[item] return r + except KeyError: + continue r = self.defaults.data.get(item) if r: diff --git a/tests/core/test_inventory.py b/tests/core/test_inventory.py index 8b3ee18a..56a90b0a 100644 --- a/tests/core/test_inventory.py +++ b/tests/core/test_inventory.py @@ -121,6 +121,7 @@ def test_var_resolution(self): assert inv.hosts["dev2.group_1"]["my_var"] == "comes_from_group_1" assert inv.hosts["dev3.group_2"]["my_var"] == "comes_from_defaults" assert inv.hosts["dev4.group_2"]["my_var"] == "comes_from_dev4.group_2" + assert inv.hosts["dev1.group_1"]["a_false_var"] is False assert inv.hosts["dev1.group_1"].data["my_var"] == "comes_from_dev1.group_1" with pytest.raises(KeyError): diff --git a/tests/inventory_data/groups.yaml b/tests/inventory_data/groups.yaml index 5196b6f5..bf996b1e 100644 --- a/tests/inventory_data/groups.yaml +++ b/tests/inventory_data/groups.yaml @@ -7,6 +7,7 @@ parent_group: platform: data: a_var: blah + a_false_var: false groups: [] connection_options: dummy: From 444e9ddf372179f1fcf39bbe4a447b2f279fe704 Mon Sep 17 00:00:00 2001 From: Omer Shtivi Date: Sat, 24 Nov 2018 19:50:36 +0200 Subject: [PATCH 105/109] Changed to is not None, changed comments, formatted to black --- CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index b5454511..e5b7e213 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -70,7 +70,7 @@ In order to run tests locally you need to have `Docker Date: Mon, 26 Nov 2018 17:01:46 -0500 Subject: [PATCH 106/109] Update grouping_tasks.ipynb Needed to add the key to access the nested value. --- docs/tutorials/intro/grouping_tasks.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/intro/grouping_tasks.ipynb b/docs/tutorials/intro/grouping_tasks.ipynb index ecf3b5df..d2e1bd9c 100644 --- a/docs/tutorials/intro/grouping_tasks.ipynb +++ b/docs/tutorials/intro/grouping_tasks.ipynb @@ -83,7 +83,7 @@ "output_type": "stream", "text": [ "hostname {{ host }}\n", - "ip domain-name {{ site }}.{{ domain }}\n", + "ip domain-name {{ host.site }}.{{ host.domain }}\n", "\u001b[0m\u001b[0m" ] } @@ -103,7 +103,7 @@ "text": [ "system {\n", " host-name {{ host }};\n", - " domain-name {{ site }}.{{ domain }};\n", + " domain-name {{ host.site }}.{{ host.domain }};\n", "}\n", "\u001b[0m\u001b[0m" ] From 522e4654ad01e33baf2af1ee0564f7e8e65e70bd Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 16 Dec 2018 15:51:25 +0100 Subject: [PATCH 107/109] bump verstion to 2.0.0 and update changelog --- CHANGELOG.rst | 16 ++++++++++++++++ setup.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index be560d23..f54da7a5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,19 @@ +2.0.0 - December 17 2018 +======================== + +For details about upgrading to 2.0.0 see the [https://nornir.readthedocs.io/en/2.0.0-beta/upgrading/1_to_2.html](notes). + ++ [CORE_ENHANCEMENTS] Lots of core enhancements, too many to document ++ [CORE_ENHANCEMENTS] Changes on how the inventory ++ [CORE_ENHANCEMENTS] New ``F`` object for advanced filtering of hosts [docs](file:///Users/dbarroso/workspace/nornir/docs/_build/html/howto/advanced_filtering.html) ++ [CORE_ENHANCEMENTS] Improvements on how to serialize/deserialize user facing data like the configuration and the inventory ++ [CORE_ENHANCEMENTS] Connections are now their own type of plugin ++ [CORE_ENHANCEMENTS] Ability to handle connections manually [docs](file:///Users/dbarroso/workspace/nornir/docs/_build/html/howto/handling_connections.html) ++ [CORE_BUGFIX] Lots ++ [PLUGIN_BUGFIX] Lots ++ [PLUGIN_NEW] netmiko_save_config ++ [PLUGIN_NEW] echo_data + 1.1.0 - July 12 2018 ==================== diff --git a/setup.py b/setup.py index 5588f733..0055495c 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ __author__ = "dbarrosop@dravetech.com" __license__ = "Apache License, version 2" -__version__ = "2.0.0b2" +__version__ = "2.0.0" setup( name="nornir", From 2c4ae55f1426afe6b3fd8742309cd395bea9ce3d Mon Sep 17 00:00:00 2001 From: David Barroso Date: Sun, 16 Dec 2018 16:38:37 +0100 Subject: [PATCH 108/109] fix CI --- .../configuration-parameters.j2 | 2 +- docs/tutorials/intro/inventory.ipynb | 101 ++++++++---------- nornir/core/inventory.py | 1 - 3 files changed, 46 insertions(+), 58 deletions(-) diff --git a/docs/_data_templates/configuration-parameters.j2 b/docs/_data_templates/configuration-parameters.j2 index e031415f..08c49d84 100644 --- a/docs/_data_templates/configuration-parameters.j2 +++ b/docs/_data_templates/configuration-parameters.j2 @@ -17,7 +17,7 @@ * - **Default** - {{ "``{}``".format(v["default"]) if v["default"] else "" }} * - **Required** - - ``{{ v["required"] }}`` + - ``{{ v["required"] or false }}`` * - **Environment Variable** - ``{{ "NORNIR_{}_{}".format(section, k).upper() }}`` diff --git a/docs/tutorials/intro/inventory.ipynb b/docs/tutorials/intro/inventory.ipynb index bcb5015a..f2c36831 100644 --- a/docs/tutorials/intro/inventory.ipynb +++ b/docs/tutorials/intro/inventory.ipynb @@ -306,87 +306,76 @@ " \"properties\": {\n", " \"hostname\": {\n", " \"title\": \"Hostname\",\n", - " \"required\": false,\n", - " \"type\": \"str\"\n", + " \"type\": \"string\"\n", " },\n", " \"port\": {\n", " \"title\": \"Port\",\n", - " \"required\": false,\n", - " \"type\": \"int\"\n", + " \"type\": \"integer\"\n", " },\n", " \"username\": {\n", " \"title\": \"Username\",\n", - " \"required\": false,\n", - " \"type\": \"str\"\n", + " \"type\": \"string\"\n", " },\n", " \"password\": {\n", " \"title\": \"Password\",\n", - " \"required\": false,\n", - " \"type\": \"str\"\n", + " \"type\": \"string\"\n", " },\n", " \"platform\": {\n", " \"title\": \"Platform\",\n", - " \"required\": false,\n", - " \"type\": \"str\"\n", + " \"type\": \"string\"\n", " },\n", " \"groups\": {\n", " \"title\": \"Groups\",\n", - " \"required\": false,\n", " \"default\": [],\n", - " \"type\": \"list\",\n", - " \"item_type\": \"str\"\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"string\"\n", + " }\n", " },\n", " \"data\": {\n", " \"title\": \"Data\",\n", - " \"required\": false,\n", " \"default\": {},\n", - " \"type\": \"mapping\",\n", - " \"item_type\": \"any\",\n", - " \"key_type\": \"str\"\n", + " \"type\": \"object\"\n", " },\n", " \"connection_options\": {\n", " \"title\": \"Connection_Options\",\n", - " \"required\": false,\n", " \"default\": {},\n", - " \"type\": \"mapping\",\n", - " \"item_type\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"hostname\": {\n", - " \"title\": \"Hostname\",\n", - " \"required\": false,\n", - " \"type\": \"str\"\n", - " },\n", - " \"port\": {\n", - " \"title\": \"Port\",\n", - " \"required\": false,\n", - " \"type\": \"int\"\n", - " },\n", - " \"username\": {\n", - " \"title\": \"Username\",\n", - " \"required\": false,\n", - " \"type\": \"str\"\n", - " },\n", - " \"password\": {\n", - " \"title\": \"Password\",\n", - " \"required\": false,\n", - " \"type\": \"str\"\n", - " },\n", - " \"platform\": {\n", - " \"title\": \"Platform\",\n", - " \"required\": false,\n", - " \"type\": \"str\"\n", - " },\n", - " \"extras\": {\n", - " \"title\": \"Extras\",\n", - " \"required\": false,\n", - " \"type\": \"mapping\",\n", - " \"item_type\": \"any\",\n", - " \"key_type\": \"str\"\n", - " }\n", + " \"type\": \"object\",\n", + " \"additionalProperties\": {\n", + " \"$ref\": \"#/definitions/ConnectionOptions\"\n", + " }\n", + " }\n", + " },\n", + " \"definitions\": {\n", + " \"ConnectionOptions\": {\n", + " \"title\": \"ConnectionOptions\",\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"hostname\": {\n", + " \"title\": \"Hostname\",\n", + " \"type\": \"string\"\n", + " },\n", + " \"port\": {\n", + " \"title\": \"Port\",\n", + " \"type\": \"integer\"\n", + " },\n", + " \"username\": {\n", + " \"title\": \"Username\",\n", + " \"type\": \"string\"\n", + " },\n", + " \"password\": {\n", + " \"title\": \"Password\",\n", + " \"type\": \"string\"\n", + " },\n", + " \"platform\": {\n", + " \"title\": \"Platform\",\n", + " \"type\": \"string\"\n", + " },\n", + " \"extras\": {\n", + " \"title\": \"Extras\",\n", + " \"type\": \"object\"\n", " }\n", - " },\n", - " \"key_type\": \"str\"\n", + " }\n", " }\n", " }\n", "}\n" diff --git a/nornir/core/inventory.py b/nornir/core/inventory.py index 56f4587d..62bdf021 100644 --- a/nornir/core/inventory.py +++ b/nornir/core/inventory.py @@ -48,7 +48,6 @@ class ParentGroups(UserList): __slots__ = "refs" def __init__(self, *args, **kwargs) -> None: - self.data: List[str] = [] super().__init__(*args, **kwargs) self.refs: List["Group"] = kwargs.get("refs", []) From 976e2640bc5f02159f9ccd438901d62e9957175c Mon Sep 17 00:00:00 2001 From: David Barroso Date: Mon, 17 Dec 2018 11:14:42 +0100 Subject: [PATCH 109/109] add echo_data plugin (#285) --- docs/plugins/tasks/data.rst | 8 + docs/plugins/tasks/data/echo_data.ipynb | 156 ++++++++ .../data/echo_data/inventory/groups.yaml | 18 + .../tasks/data/echo_data/inventory/hosts.yaml | 78 ++++ .../tutorials/intro/initializing_nornir.ipynb | 18 +- docs/tutorials/intro/inventory.ipynb | 342 +++++++++--------- nornir/plugins/tasks/data/__init__.py | 3 +- nornir/plugins/tasks/data/echo_data.py | 16 + requirements-pinned.txt | 113 ++++++ tox.ini | 3 +- 10 files changed, 573 insertions(+), 182 deletions(-) create mode 100644 docs/plugins/tasks/data/echo_data.ipynb create mode 100644 docs/plugins/tasks/data/echo_data/inventory/groups.yaml create mode 100644 docs/plugins/tasks/data/echo_data/inventory/hosts.yaml create mode 100644 nornir/plugins/tasks/data/echo_data.py create mode 100644 requirements-pinned.txt diff --git a/docs/plugins/tasks/data.rst b/docs/plugins/tasks/data.rst index 0c93380a..6a142405 100644 --- a/docs/plugins/tasks/data.rst +++ b/docs/plugins/tasks/data.rst @@ -1,6 +1,14 @@ Data ==== +.. toctree:: + :maxdepth: 1 + :glob: + + data/* + +Old-style docs: + .. automodule:: nornir.plugins.tasks.data :members: :undoc-members: diff --git a/docs/plugins/tasks/data/echo_data.ipynb b/docs/plugins/tasks/data/echo_data.ipynb new file mode 100644 index 00000000..3987976c --- /dev/null +++ b/docs/plugins/tasks/data/echo_data.ipynb @@ -0,0 +1,156 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# echo_data" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " Dummy task that echoes the data passed to it. Useful in grouped_tasks\n", + " to debug data passed to tasks.\n", + "\n", + " Arguments:\n", + " ``**kwargs``: Any pair you want\n", + "\n", + " Returns:\n", + " Result object with the following attributes set:\n", + " * result (``dict``): ``**kwargs`` passed to the task\n", + " \n" + ] + } + ], + "source": [ + "from nornir.plugins.tasks.data import echo_data\n", + "\n", + "print(echo_data.__doc__)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1m\u001b[36mgrouped_task********************************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* bat ** changed : False *******************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32mvvvv grouped_task ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m---- bat ** changed : False ---------------------------------------------------- INFO\u001b[0m\n", + "\u001b[0m{\u001b[0m \u001b[0m'diet'\u001b[0m: \u001b[0m'carnivore'\u001b[0m,\n", + " \u001b[0m'even_complex'\u001b[0m: \u001b[0m{'a': 1, 'b': 2, 'c': 'asd'}\u001b[0m,\n", + " \u001b[0m'famous_members'\u001b[0m: \u001b[0m['batman', 'count chocula', 'nosferatu']\u001b[0m,\n", + " \u001b[0m'more_data'\u001b[0m: \u001b[0m'whatever you want'\u001b[0m}\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m^^^^ END grouped_task ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* canary ** changed : False ****************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32mvvvv grouped_task ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m---- canary ** changed : False ------------------------------------------------- INFO\u001b[0m\n", + "\u001b[0m{\u001b[0m \u001b[0m'diet'\u001b[0m: \u001b[0m'herbivore'\u001b[0m,\n", + " \u001b[0m'even_complex'\u001b[0m: \u001b[0m{'a': 1, 'b': 2, 'c': 'asd'}\u001b[0m,\n", + " \u001b[0m'famous_members'\u001b[0m: \u001b[0m['tweetie']\u001b[0m,\n", + " \u001b[0m'more_data'\u001b[0m: \u001b[0m'whatever you want'\u001b[0m}\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m^^^^ END grouped_task ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* cat ** changed : False *******************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32mvvvv grouped_task ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m---- cat ** changed : False ---------------------------------------------------- INFO\u001b[0m\n", + "\u001b[0m{\u001b[0m \u001b[0m'diet'\u001b[0m: \u001b[0m'omnivore'\u001b[0m,\n", + " \u001b[0m'even_complex'\u001b[0m: \u001b[0m{'a': 1, 'b': 2, 'c': 'asd'}\u001b[0m,\n", + " \u001b[0m'famous_members'\u001b[0m: \u001b[0m['garfield', 'felix', 'grumpy']\u001b[0m,\n", + " \u001b[0m'more_data'\u001b[0m: \u001b[0m'whatever you want'\u001b[0m}\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m^^^^ END grouped_task ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* caterpillaer ** changed : False **********************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32mvvvv grouped_task ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m---- caterpillaer ** changed : False ------------------------------------------- INFO\u001b[0m\n", + "\u001b[0m{\u001b[0m \u001b[0m'diet'\u001b[0m: \u001b[0m'herbivore'\u001b[0m,\n", + " \u001b[0m'even_complex'\u001b[0m: \u001b[0m{'a': 1, 'b': 2, 'c': 'asd'}\u001b[0m,\n", + " \u001b[0m'famous_members'\u001b[0m: \u001b[0m['Hookah-Smoking']\u001b[0m,\n", + " \u001b[0m'more_data'\u001b[0m: \u001b[0m'whatever you want'\u001b[0m}\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m^^^^ END grouped_task ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* eagle ** changed : False *****************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32mvvvv grouped_task ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m---- eagle ** changed : False -------------------------------------------------- INFO\u001b[0m\n", + "\u001b[0m{\u001b[0m \u001b[0m'diet'\u001b[0m: \u001b[0m'carnivore'\u001b[0m,\n", + " \u001b[0m'even_complex'\u001b[0m: \u001b[0m{'a': 1, 'b': 2, 'c': 'asd'}\u001b[0m,\n", + " \u001b[0m'famous_members'\u001b[0m: \u001b[0m['thorondor', 'sam']\u001b[0m,\n", + " \u001b[0m'more_data'\u001b[0m: \u001b[0m'whatever you want'\u001b[0m}\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m^^^^ END grouped_task ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[34m* octopus ** changed : False ***************************************************\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32mvvvv grouped_task ** changed : False vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv INFO\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m---- octopus ** changed : False ------------------------------------------------ INFO\u001b[0m\n", + "\u001b[0m{\u001b[0m \u001b[0m'diet'\u001b[0m: \u001b[0m'carnivore'\u001b[0m,\n", + " \u001b[0m'even_complex'\u001b[0m: \u001b[0m{'a': 1, 'b': 2, 'c': 'asd'}\u001b[0m,\n", + " \u001b[0m'famous_members'\u001b[0m: \u001b[0m['sharktopus']\u001b[0m,\n", + " \u001b[0m'more_data'\u001b[0m: \u001b[0m'whatever you want'\u001b[0m}\u001b[0m\n", + "\u001b[0m\u001b[1m\u001b[32m^^^^ END grouped_task ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", + "\u001b[0m" + ] + } + ], + "source": [ + "from nornir import InitNornir\n", + "from nornir.plugins.tasks.data import echo_data\n", + "from nornir.plugins.functions.text import print_result\n", + "\n", + "nr = InitNornir(\n", + " inventory={\n", + " \"options\": {\n", + " \"host_file\": \"echo_data/inventory/hosts.yaml\",\n", + " \"group_file\": \"echo_data/inventory/groups.yaml\"\n", + " }\n", + " }\n", + ")\n", + "\n", + "def grouped_task(task):\n", + " task.run(task=echo_data,\n", + " name=task.host.name,\n", + " diet=task.host[\"diet\"],\n", + " famous_members=task.host[\"additional_data\"][\"famous_members\"],\n", + " more_data=\"whatever you want\",\n", + " even_complex={\"a\": 1, \"b\": 2, \"c\": \"asd\"})\n", + " \n", + "r = nr.run(task=grouped_task)\n", + "print_result(r)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/plugins/tasks/data/echo_data/inventory/groups.yaml b/docs/plugins/tasks/data/echo_data/inventory/groups.yaml new file mode 100644 index 00000000..175bae52 --- /dev/null +++ b/docs/plugins/tasks/data/echo_data/inventory/groups.yaml @@ -0,0 +1,18 @@ +--- +mammal: + data: + reproduction: birth + fly: false + +bird: + data: + reproduction: eggs + fly: true + +invertebrate: + data: + reproduction: mitosis + fly: false + +terrestrial: {} +marine: {} diff --git a/docs/plugins/tasks/data/echo_data/inventory/hosts.yaml b/docs/plugins/tasks/data/echo_data/inventory/hosts.yaml new file mode 100644 index 00000000..e5d3acbb --- /dev/null +++ b/docs/plugins/tasks/data/echo_data/inventory/hosts.yaml @@ -0,0 +1,78 @@ +--- +cat: + groups: + - terrestrial + - mammal + data: + domestic: true + diet: omnivore + additional_data: + lifespan: 17 + famous_members: + - garfield + - felix + - grumpy + +bat: + groups: + - terrestrial + - mammal + data: + domestic: false + fly: true + diet: carnivore + additional_data: + lifespan: 15 + famous_members: + - batman + - count chocula + - nosferatu + +eagle: + groups: + - terrestrial + - bird + data: + domestic: false + diet: carnivore + additional_data: + lifespan: 50 + famous_members: + - thorondor + - sam + +canary: + groups: + - terrestrial + - bird + data: + domestic: true + diet: herbivore + additional_data: + lifespan: 15 + famous_members: + - tweetie + +caterpillaer: + groups: + - terrestrial + - invertebrate + data: + domestic: false + diet: herbivore + additional_data: + lifespan: 1 + famous_members: + - Hookah-Smoking + +octopus: + groups: + - marine + - invertebrate + data: + domestic: false + diet: carnivore + additional_data: + lifespan: 1 + famous_members: + - sharktopus diff --git a/docs/tutorials/intro/initializing_nornir.ipynb b/docs/tutorials/intro/initializing_nornir.ipynb index bbfe2696..5f077eab 100644 --- a/docs/tutorials/intro/initializing_nornir.ipynb +++ b/docs/tutorials/intro/initializing_nornir.ipynb @@ -108,15 +108,15 @@ "\n", "\n", "
 1 ---\n",
-       " 2 core:\n",
-       " 3     num_workers: 100\n",
+       " 2 core:\n",
+       " 3     num_workers: 100\n",
        " 4 \n",
-       " 5 inventory:\n",
-       " 6     plugin: nornir.plugins.inventory.simple.SimpleInventory\n",
-       " 7     options:\n",
-       " 8         host_file: "inventory/hosts.yaml"\n",
-       " 9         group_file: "inventory/groups.yaml"\n",
-       "10         defaults_file: "inventory/defaults.yaml"\n",
+       " 5 inventory:\n",
+       " 6     plugin: nornir.plugins.inventory.simple.SimpleInventory\n",
+       " 7     options:\n",
+       " 8         host_file: "inventory/hosts.yaml"\n",
+       " 9         group_file: "inventory/groups.yaml"\n",
+       "10         defaults_file: "inventory/defaults.yaml"\n",
        "
\n", "\n" ], @@ -230,7 +230,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.0" + "version": "3.7.1" } }, "nbformat": 4, diff --git a/docs/tutorials/intro/inventory.ipynb b/docs/tutorials/intro/inventory.ipynb index f2c36831..8cd900fd 100644 --- a/docs/tutorials/intro/inventory.ipynb +++ b/docs/tutorials/intro/inventory.ipynb @@ -106,167 +106,167 @@ "\n", "\n", "
  1 ---\n",
-       "  2 host1.cmh:\n",
-       "  3     hostname: 127.0.0.1\n",
-       "  4     port: 2201\n",
-       "  5     username: vagrant\n",
-       "  6     password: vagrant\n",
-       "  7     platform: linux\n",
-       "  8     groups:\n",
+       "  2 host1.cmh:\n",
+       "  3     hostname: 127.0.0.1\n",
+       "  4     port: 2201\n",
+       "  5     username: vagrant\n",
+       "  6     password: vagrant\n",
+       "  7     platform: linux\n",
+       "  8     groups:\n",
        "  9         - cmh\n",
-       " 10     data:\n",
-       " 11         site: cmh\n",
-       " 12         role: host\n",
-       " 13         type: host\n",
-       " 14         nested_data:\n",
-       " 15             a_dict:\n",
-       " 16                 a: 1\n",
-       " 17                 b: 2\n",
-       " 18             a_list: [1, 2]\n",
-       " 19             a_string: "asdasd"\n",
+       " 10     data:\n",
+       " 11         site: cmh\n",
+       " 12         role: host\n",
+       " 13         type: host\n",
+       " 14         nested_data:\n",
+       " 15             a_dict:\n",
+       " 16                 a: 1\n",
+       " 17                 b: 2\n",
+       " 18             a_list: [1, 2]\n",
+       " 19             a_string: "asdasd"\n",
        " 20 \n",
-       " 21 host2.cmh:\n",
-       " 22     hostname: 127.0.0.1\n",
-       " 23     port: 2202\n",
-       " 24     username: vagrant\n",
-       " 25     password: vagrant\n",
-       " 26     platform: linux\n",
-       " 27     groups:\n",
+       " 21 host2.cmh:\n",
+       " 22     hostname: 127.0.0.1\n",
+       " 23     port: 2202\n",
+       " 24     username: vagrant\n",
+       " 25     password: vagrant\n",
+       " 26     platform: linux\n",
+       " 27     groups:\n",
        " 28         - cmh\n",
-       " 29     data:\n",
-       " 30         site: cmh\n",
-       " 31         role: host\n",
-       " 32         type: host\n",
-       " 33         nested_data:\n",
-       " 34             a_dict:\n",
-       " 35                 b: 2\n",
-       " 36                 c: 3\n",
-       " 37             a_list: [1, 2]\n",
-       " 38             a_string: "qwe"\n",
+       " 29     data:\n",
+       " 30         site: cmh\n",
+       " 31         role: host\n",
+       " 32         type: host\n",
+       " 33         nested_data:\n",
+       " 34             a_dict:\n",
+       " 35                 b: 2\n",
+       " 36                 c: 3\n",
+       " 37             a_list: [1, 2]\n",
+       " 38             a_string: "qwe"\n",
        " 39 \n",
-       " 40 spine00.cmh:\n",
-       " 41     hostname: 127.0.0.1\n",
-       " 42     username: vagrant\n",
-       " 43     password: vagrant\n",
-       " 44     port: 12444\n",
-       " 45     platform: eos\n",
-       " 46     groups:\n",
+       " 40 spine00.cmh:\n",
+       " 41     hostname: 127.0.0.1\n",
+       " 42     username: vagrant\n",
+       " 43     password: vagrant\n",
+       " 44     port: 12444\n",
+       " 45     platform: eos\n",
+       " 46     groups:\n",
        " 47         - cmh\n",
-       " 48     data:\n",
-       " 49         site: cmh\n",
-       " 50         role: spine\n",
-       " 51         type: network_device\n",
+       " 48     data:\n",
+       " 49         site: cmh\n",
+       " 50         role: spine\n",
+       " 51         type: network_device\n",
        " 52 \n",
-       " 53 spine01.cmh:\n",
-       " 54     hostname: 127.0.0.1\n",
-       " 55     username: vagrant\n",
-       " 56     password: ""\n",
-       " 57     platform: junos\n",
-       " 58     port: 12204\n",
-       " 59     groups:\n",
+       " 53 spine01.cmh:\n",
+       " 54     hostname: 127.0.0.1\n",
+       " 55     username: vagrant\n",
+       " 56     password: ""\n",
+       " 57     platform: junos\n",
+       " 58     port: 12204\n",
+       " 59     groups:\n",
        " 60         - cmh\n",
-       " 61     data:\n",
-       " 62         site: cmh\n",
-       " 63         role: spine\n",
-       " 64         type: network_device\n",
+       " 61     data:\n",
+       " 62         site: cmh\n",
+       " 63         role: spine\n",
+       " 64         type: network_device\n",
        " 65 \n",
-       " 66 leaf00.cmh:\n",
-       " 67     hostname: 127.0.0.1\n",
-       " 68     username: vagrant\n",
-       " 69     password: vagrant\n",
-       " 70     port: 12443\n",
-       " 71     platform: eos\n",
-       " 72     groups:\n",
+       " 66 leaf00.cmh:\n",
+       " 67     hostname: 127.0.0.1\n",
+       " 68     username: vagrant\n",
+       " 69     password: vagrant\n",
+       " 70     port: 12443\n",
+       " 71     platform: eos\n",
+       " 72     groups:\n",
        " 73         - cmh\n",
-       " 74     data:\n",
-       " 75         site: cmh\n",
-       " 76         role: leaf\n",
-       " 77         type: network_device\n",
-       " 78         asn: 65100\n",
+       " 74     data:\n",
+       " 75         site: cmh\n",
+       " 76         role: leaf\n",
+       " 77         type: network_device\n",
+       " 78         asn: 65100\n",
        " 79 \n",
-       " 80 leaf01.cmh:\n",
-       " 81     hostname: 127.0.0.1\n",
-       " 82     username: vagrant\n",
-       " 83     password: ""\n",
-       " 84     port: 12203\n",
-       " 85     platform: junos\n",
-       " 86     groups:\n",
+       " 80 leaf01.cmh:\n",
+       " 81     hostname: 127.0.0.1\n",
+       " 82     username: vagrant\n",
+       " 83     password: ""\n",
+       " 84     port: 12203\n",
+       " 85     platform: junos\n",
+       " 86     groups:\n",
        " 87         - cmh\n",
-       " 88     data:\n",
-       " 89         site: cmh\n",
-       " 90         role: leaf\n",
-       " 91         type: network_device\n",
-       " 92         asn: 65101\n",
+       " 88     data:\n",
+       " 89         site: cmh\n",
+       " 90         role: leaf\n",
+       " 91         type: network_device\n",
+       " 92         asn: 65101\n",
        " 93 \n",
-       " 94 host1.bma:\n",
-       " 95     groups:\n",
+       " 94 host1.bma:\n",
+       " 95     groups:\n",
        " 96         - bma\n",
-       " 97     platform: linux\n",
-       " 98     data:\n",
-       " 99         site: bma\n",
-       "100         role: host\n",
-       "101         type: host\n",
+       " 97     platform: linux\n",
+       " 98     data:\n",
+       " 99         site: bma\n",
+       "100         role: host\n",
+       "101         type: host\n",
        "102 \n",
-       "103 host2.bma:\n",
-       "104     groups:\n",
+       "103 host2.bma:\n",
+       "104     groups:\n",
        "105         - bma\n",
-       "106     platform: linux\n",
-       "107     data:\n",
-       "108         site: bma\n",
-       "109         role: host\n",
-       "110         type: host\n",
+       "106     platform: linux\n",
+       "107     data:\n",
+       "108         site: bma\n",
+       "109         role: host\n",
+       "110         type: host\n",
        "111 \n",
-       "112 spine00.bma:\n",
-       "113     hostname: 127.0.0.1\n",
-       "114     username: vagrant\n",
-       "115     password: vagrant\n",
-       "116     port: 12444\n",
-       "117     platform: eos\n",
-       "118     groups:\n",
+       "112 spine00.bma:\n",
+       "113     hostname: 127.0.0.1\n",
+       "114     username: vagrant\n",
+       "115     password: vagrant\n",
+       "116     port: 12444\n",
+       "117     platform: eos\n",
+       "118     groups:\n",
        "119         - bma\n",
-       "120     data:\n",
-       "121         site: bma\n",
-       "122         role: spine\n",
-       "123         type: network_device\n",
+       "120     data:\n",
+       "121         site: bma\n",
+       "122         role: spine\n",
+       "123         type: network_device\n",
        "124 \n",
-       "125 spine01.bma:\n",
-       "126     hostname: 127.0.0.1\n",
-       "127     username: vagrant\n",
-       "128     password: ""\n",
-       "129     port: 12204\n",
-       "130     platform: junos\n",
-       "131     groups:\n",
+       "125 spine01.bma:\n",
+       "126     hostname: 127.0.0.1\n",
+       "127     username: vagrant\n",
+       "128     password: ""\n",
+       "129     port: 12204\n",
+       "130     platform: junos\n",
+       "131     groups:\n",
        "132         - bma\n",
-       "133     data:\n",
-       "134         site: bma\n",
-       "135         role: spine\n",
-       "136         type: network_device\n",
+       "133     data:\n",
+       "134         site: bma\n",
+       "135         role: spine\n",
+       "136         type: network_device\n",
        "137 \n",
-       "138 leaf00.bma:\n",
-       "139     hostname: 127.0.0.1\n",
-       "140     username: vagrant\n",
-       "141     password: vagrant\n",
-       "142     port: 12443\n",
-       "143     platform: eos\n",
-       "144     groups:\n",
+       "138 leaf00.bma:\n",
+       "139     hostname: 127.0.0.1\n",
+       "140     username: vagrant\n",
+       "141     password: vagrant\n",
+       "142     port: 12443\n",
+       "143     platform: eos\n",
+       "144     groups:\n",
        "145         - bma\n",
-       "146     data:\n",
-       "147         site: bma\n",
-       "148         role: leaf\n",
-       "149         type: network_device\n",
+       "146     data:\n",
+       "147         site: bma\n",
+       "148         role: leaf\n",
+       "149         type: network_device\n",
        "150 \n",
-       "151 leaf01.bma:\n",
-       "152     hostname: 127.0.0.1\n",
-       "153     username: vagrant\n",
-       "154     password: wrong_password\n",
-       "155     port: 12203\n",
-       "156     platform: junos\n",
-       "157     groups:\n",
+       "151 leaf01.bma:\n",
+       "152     hostname: 127.0.0.1\n",
+       "153     username: vagrant\n",
+       "154     password: wrong_password\n",
+       "155     port: 12203\n",
+       "156     platform: junos\n",
+       "157     groups:\n",
        "158         - bma\n",
-       "159     data:\n",
-       "160         site: bma\n",
-       "161         role: leaf\n",
-       "162         type: network_device\n",
+       "159     data:\n",
+       "160         site: bma\n",
+       "161         role: leaf\n",
+       "162         type: network_device\n",
        "
\n", "\n" ], @@ -478,26 +478,26 @@ "\n", "\n", "
 1 ---\n",
-       " 2 global:\n",
-       " 3     data:\n",
-       " 4         domain: global.local\n",
-       " 5         asn: 1\n",
+       " 2 global:\n",
+       " 3     data:\n",
+       " 4         domain: global.local\n",
+       " 5         asn: 1\n",
        " 6 \n",
-       " 7 eu:\n",
-       " 8     data:\n",
-       " 9         asn: 65100\n",
+       " 7 eu:\n",
+       " 8     data:\n",
+       " 9         asn: 65100\n",
        "10 \n",
-       "11 bma:\n",
-       "12     groups:\n",
+       "11 bma:\n",
+       "12     groups:\n",
        "13         - eu\n",
        "14         - global\n",
        "15 \n",
-       "16 cmh:\n",
-       "17     data:\n",
-       "18         asn: 65000\n",
-       "19         vlans:\n",
-       "20           100: frontend\n",
-       "21           200: backend\n",
+       "16 cmh:\n",
+       "17     data:\n",
+       "18         asn: 65000\n",
+       "19         vlans:\n",
+       "20           100: frontend\n",
+       "21           200: backend\n",
        "
\n", "\n" ], @@ -605,8 +605,8 @@ "\n", "\n", "
1 ---\n",
-       "2 data:\n",
-       "3     domain: acme.local\n",
+       "2 data:\n",
+       "3     domain: acme.local\n",
        "
\n", "\n" ], @@ -876,26 +876,26 @@ "\n", "\n", "
 1 ---\n",
-       " 2 global:\n",
-       " 3     data:\n",
-       " 4         domain: global.local\n",
-       " 5         asn: 1\n",
+       " 2 global:\n",
+       " 3     data:\n",
+       " 4         domain: global.local\n",
+       " 5         asn: 1\n",
        " 6 \n",
-       " 7 eu:\n",
-       " 8     data:\n",
-       " 9         asn: 65100\n",
+       " 7 eu:\n",
+       " 8     data:\n",
+       " 9         asn: 65100\n",
        "10 \n",
-       "11 bma:\n",
-       "12     groups:\n",
+       "11 bma:\n",
+       "12     groups:\n",
        "13         - eu\n",
        "14         - global\n",
        "15 \n",
-       "16 cmh:\n",
-       "17     data:\n",
-       "18         asn: 65000\n",
-       "19         vlans:\n",
-       "20           100: frontend\n",
-       "21           200: backend\n",
+       "16 cmh:\n",
+       "17     data:\n",
+       "18         asn: 65000\n",
+       "19         vlans:\n",
+       "20           100: frontend\n",
+       "21           200: backend\n",
        "
\n", "\n" ], @@ -1451,7 +1451,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.0" + "version": "3.7.1" } }, "nbformat": 4, diff --git a/nornir/plugins/tasks/data/__init__.py b/nornir/plugins/tasks/data/__init__.py index 2dbc5c33..b0528cee 100644 --- a/nornir/plugins/tasks/data/__init__.py +++ b/nornir/plugins/tasks/data/__init__.py @@ -1,5 +1,6 @@ from .load_json import load_json from .load_yaml import load_yaml +from .echo_data import echo_data -__all__ = ("load_json", "load_yaml") +__all__ = ("load_json", "load_yaml", "echo_data") diff --git a/nornir/plugins/tasks/data/echo_data.py b/nornir/plugins/tasks/data/echo_data.py new file mode 100644 index 00000000..d225e3b6 --- /dev/null +++ b/nornir/plugins/tasks/data/echo_data.py @@ -0,0 +1,16 @@ +from nornir.core.task import Result, Task + + +def echo_data(task: Task, **kwargs): + """ + Dummy task that echoes the data passed to it. Useful in grouped_tasks + to debug data passed to tasks. + + Arguments: + ``**kwargs``: Any pair you want + + Returns: + Result object with the following attributes set: + * result (``dict``): ``**kwargs`` passed to the task + """ + return Result(host=task.host, result=kwargs) diff --git a/requirements-pinned.txt b/requirements-pinned.txt new file mode 100644 index 00000000..3f07827b --- /dev/null +++ b/requirements-pinned.txt @@ -0,0 +1,113 @@ +alabaster==0.7.12 +appdirs==1.4.3 +appnope==0.1.0 +asn1crypto==0.24.0 +atomicwrites==1.2.1 +attrs==18.2.0 +Babel==2.6.0 +backcall==0.1.0 +bcrypt==3.1.5 +black==18.6b4 +bleach==3.0.2 +certifi==2018.11.29 +cffi==1.11.5 +chardet==3.0.4 +Click==7.0 +colorama==0.4.1 +coverage==4.5.2 +cryptography==2.4.2 +decorator==4.3.0 +defusedxml==0.5.0 +docutils==0.14 +entrypoints==0.2.3 +filelock==3.0.10 +flake8-import-order==0.18 +future==0.17.1 +idna==2.8 +imagesize==1.1.0 +ipykernel==5.1.0 +ipython==7.2.0 +ipython-genutils==0.2.0 +ipywidgets==7.4.2 +jedi==0.13.2 +Jinja2==2.10 +jsonschema==2.6.0 +junos-eznc==2.2.0 +jupyter==1.0.0 +jupyter-client==5.2.4 +jupyter-console==6.0.0 +jupyter-core==4.4.0 +lxml==4.2.5 +MarkupSafe==1.1.0 +mccabe==0.6.1 +mistune==0.8.4 +more-itertools==4.3.0 +mypy==0.650 +mypy-extensions==0.4.1 +napalm==2.3.3 +nbconvert==5.4.0 +nbformat==4.4.0 +nbsphinx==0.4.1 +nbval==0.9.1 +ncclient==0.6.3 +netaddr==0.7.19 +netmiko==2.3.0 +notebook==5.7.3 +packaging==18.0 +pandocfilters==1.4.2 +paramiko==2.4.2 +parso==0.3.1 +pexpect==4.6.0 +pickleshare==0.7.5 +pluggy==0.8.0 +pockets==0.7.2 +prometheus-client==0.5.0 +prompt-toolkit==2.0.7 +ptyprocess==0.6.0 +py==1.7.0 +pyasn1==0.4.4 +pycodestyle==2.4.0 +pycparser==2.19 +pydantic==0.16.1 +pydocstyle==3.0.0 +pyeapi==0.8.2 +pyflakes==2.0.0 +Pygments==2.3.1 +pyIOSXR==0.53 +pylama==7.6.6 +PyNaCl==1.3.0 +pynxos==0.0.3 +pyparsing==2.3.0 +pyserial==3.4 +pytest==4.0.2 +pytest-cov==2.6.0 +python-dateutil==2.7.5 +pytz==2018.7 +PyYAML==3.13 +pyzmq==17.1.2 +qtconsole==4.4.3 +requests==2.21.0 +requests-mock==1.5.2 +ruamel.yaml==0.15.81 +scp==0.13.0 +selectors2==2.0.1 +Send2Trash==1.5.0 +six==1.12.0 +snowballstemmer==1.2.1 +Sphinx==1.8.2 +sphinx-rtd-theme==0.4.2 +sphinxcontrib-napoleon==0.7 +sphinxcontrib-websupport==1.1.0 +terminado==0.8.1 +testpath==0.4.2 +textfsm==0.4.1 +toml==0.10.0 +tornado==5.1.1 +tox==3.6.0 +traitlets==4.3.2 +typed-ast==1.1.0 +urllib3==1.24.1 +virtualenv==16.1.0 +wcwidth==0.1.7 +webencodings==0.5.1 +widgetsnbextension==3.4.2 diff --git a/tox.ini b/tox.ini index 72cdc8c5..19ab03ec 100644 --- a/tox.ini +++ b/tox.ini @@ -44,10 +44,11 @@ commands = [testenv:nbval] deps = - -rrequirements-dev.txt + -rrequirements-pinned.txt basepython = python3.6 commands = + pytest --nbval docs/plugins pytest --nbval docs/howto pytest --nbval docs/tutorials/intro/initializing_nornir.ipynb pytest --nbval docs/tutorials/intro/inventory.ipynb