From b58056da2f5a12e3d614650904039c0655ce1221 Mon Sep 17 00:00:00 2001 From: Alexander Graul Date: Tue, 18 Jan 2022 19:41:03 +0100 Subject: [PATCH] Improvements on "ansiblegate" module (#354) * Allow collecting Ansible Inventory from a minion * Prevent crashing if ansible-playbook doesn't return JSON * Add new 'ansible.discover_playbooks' method * Include custom inventory when discovering Ansible playbooks * Enhance 'ansible.discover_playbooks' to accept a list of locations * Remove unused constants from Ansible utils * Avoid string concatenation to calculate extra cmd args * Add unit test for ansible.targets * Improve Ansible roster targetting * Add tests for new ansiblegate module functions * Fix issue dealing with ungrouped targets on inventory * Enable ansible utils for ansible roster tests * Remove unnecessary code from Ansible utils * Fix pylint issue * Fix issue in documentation Fix issue parsing errors in ansiblegate state module --- salt/modules/ansiblegate.py | 167 +++++++++++++++++- salt/roster/ansible.py | 17 +- salt/states/ansiblegate.py | 12 +- salt/utils/ansible.py | 41 +++++ .../pytests/unit/modules/test_ansiblegate.py | 99 ++++++++++- .../example_playbooks/example-playbook2/hosts | 7 + .../example-playbook2/site.yml | 28 +++ .../playbooks/example_playbooks/playbook1.yml | 5 + tests/unit/roster/test_ansible.py | 2 +- 9 files changed, 367 insertions(+), 11 deletions(-) create mode 100644 salt/utils/ansible.py create mode 100644 tests/unit/files/playbooks/example_playbooks/example-playbook2/hosts create mode 100644 tests/unit/files/playbooks/example_playbooks/example-playbook2/site.yml create mode 100644 tests/unit/files/playbooks/example_playbooks/playbook1.yml diff --git a/salt/modules/ansiblegate.py b/salt/modules/ansiblegate.py index 328d9b7b0a..f33be6a00e 100644 --- a/salt/modules/ansiblegate.py +++ b/salt/modules/ansiblegate.py @@ -17,6 +17,7 @@ any Ansible module to respond. import fnmatch import json import logging +import os import subprocess import sys from tempfile import NamedTemporaryFile @@ -365,7 +366,171 @@ def playbooks( } ret = __salt__["cmd.run_all"](**cmd_kwargs) log.debug("Ansible Playbook Return: %s", ret) - retdata = json.loads(ret["stdout"]) + try: + retdata = json.loads(ret["stdout"]) + except ValueError: + retdata = ret if "retcode" in ret: __context__["retcode"] = retdata["retcode"] = ret["retcode"] return retdata + + +def targets(**kwargs): + """ + Return the inventory from an Ansible inventory_file + + :param inventory: + The inventory file to read the inventory from. Default: "/etc/ansible/hosts" + + :param yaml: + Return the inventory as yaml output. Default: False + + :param export: + Return inventory as export format. Default: False + + CLI Example: + + .. code-block:: bash + + salt 'ansiblehost' ansible.targets + salt 'ansiblehost' ansible.targets inventory=my_custom_inventory + + """ + return __utils__["ansible.targets"](**kwargs) + + +def discover_playbooks(path=None, + locations=None, + playbook_extension=None, + hosts_filename=None, + syntax_check=False): + """ + Discover Ansible playbooks stored under the given path or from multiple paths (locations) + + This will search for files matching with the playbook file extension under the given + root path and will also look for files inside the first level of directories in this path. + + The return of this function would be a dict like this: + + .. code-block:: python + + { + "/home/foobar/": { + "my_ansible_playbook.yml": { + "fullpath": "/home/foobar/playbooks/my_ansible_playbook.yml", + "custom_inventory": "/home/foobar/playbooks/hosts" + }, + "another_playbook.yml": { + "fullpath": "/home/foobar/playbooks/another_playbook.yml", + "custom_inventory": "/home/foobar/playbooks/hosts" + }, + "lamp_simple/site.yml": { + "fullpath": "/home/foobar/playbooks/lamp_simple/site.yml", + "custom_inventory": "/home/foobar/playbooks/lamp_simple/hosts" + }, + "lamp_proxy/site.yml": { + "fullpath": "/home/foobar/playbooks/lamp_proxy/site.yml", + "custom_inventory": "/home/foobar/playbooks/lamp_proxy/hosts" + } + }, + "/srv/playbooks/": { + "example_playbook/example.yml": { + "fullpath": "/srv/playbooks/example_playbook/example.yml", + "custom_inventory": "/srv/playbooks/example_playbook/hosts" + } + } + } + + :param path: + Path to discover playbooks from. + + :param locations: + List of paths to discover playbooks from. + + :param playbook_extension: + File extension of playbooks file to search for. Default: "yml" + + :param hosts_filename: + Filename of custom playbook inventory to search for. Default: "hosts" + + :param syntax_check: + Skip playbooks that do not pass "ansible-playbook --syntax-check" validation. Default: False + + :return: + The discovered playbooks under the given paths + + CLI Example: + + .. code-block:: bash + + salt 'ansiblehost' ansible.discover_playbooks path=/srv/playbooks/ + salt 'ansiblehost' ansible.discover_playbooks locations='["/srv/playbooks/", "/srv/foobar"]' + + """ + + if not path and not locations: + raise CommandExecutionError("You have to specify either 'path' or 'locations' arguments") + + if path and locations: + raise CommandExecutionError("You cannot specify 'path' and 'locations' at the same time") + + if not playbook_extension: + playbook_extension = "yml" + if not hosts_filename: + hosts_filename = "hosts" + + if path: + if not os.path.isabs(path): + raise CommandExecutionError("The given path is not an absolute path: {}".format(path)) + if not os.path.isdir(path): + raise CommandExecutionError("The given path is not a directory: {}".format(path)) + return {path: _explore_path(path, playbook_extension, hosts_filename, syntax_check)} + + if locations: + all_ret = {} + for location in locations: + all_ret[location] = _explore_path(location, playbook_extension, hosts_filename, syntax_check) + return all_ret + + +def _explore_path(path, playbook_extension, hosts_filename, syntax_check): + ret = {} + + if not os.path.isabs(path): + log.error("The given path is not an absolute path: {}".format(path)) + return ret + if not os.path.isdir(path): + log.error("The given path is not a directory: {}".format(path)) + return ret + + try: + # Check files in the given path + for _f in os.listdir(path): + _path = os.path.join(path, _f) + if os.path.isfile(_path) and _path.endswith("." + playbook_extension): + ret[_f] = {"fullpath": _path} + # Check for custom inventory file + if os.path.isfile(os.path.join(path, hosts_filename)): + ret[_f].update({"custom_inventory": os.path.join(path, hosts_filename)}) + elif os.path.isdir(_path): + # Check files in the 1st level of subdirectories + for _f2 in os.listdir(_path): + _path2 = os.path.join(_path, _f2) + if os.path.isfile(_path2) and _path2.endswith("." + playbook_extension): + ret[os.path.join(_f, _f2)] = {"fullpath": _path2} + # Check for custom inventory file + if os.path.isfile(os.path.join(_path, hosts_filename)): + ret[os.path.join(_f, _f2)].update({"custom_inventory": os.path.join(_path, hosts_filename)}) + except Exception as exc: + raise CommandExecutionError("There was an exception while discovering playbooks: {}".format(exc)) + + # Run syntax check validation + if syntax_check: + check_command = ["ansible-playbook", "--syntax-check"] + try: + for pb in list(ret): + if __salt__["cmd.retcode"](check_command + [ret[pb]]): + del ret[pb] + except Exception as exc: + raise CommandExecutionError("There was an exception while checking syntax of playbooks: {}".format(exc)) + return ret diff --git a/salt/roster/ansible.py b/salt/roster/ansible.py index 7beaaf2075..d3b352de27 100644 --- a/salt/roster/ansible.py +++ b/salt/roster/ansible.py @@ -117,27 +117,32 @@ def targets(tgt, tgt_type="glob", **kwargs): Return the targets from the ansible inventory_file Default: /etc/salt/roster """ - inventory = __runner__["salt.cmd"]( - "cmd.run", "ansible-inventory -i {} --list".format(get_roster_file(__opts__)) - ) - __context__["inventory"] = __utils__["json.loads"]( - __utils__["stringutils.to_str"](inventory) + __context__["inventory"] = __utils__["ansible.targets"]( + inventory=get_roster_file(__opts__), **kwargs ) if tgt_type == "glob": hosts = [ host for host in _get_hosts_from_group("all") if fnmatch.fnmatch(host, tgt) ] + elif tgt_type == "list": + hosts = [host for host in _get_hosts_from_group("all") if host in tgt] elif tgt_type == "nodegroup": hosts = _get_hosts_from_group(tgt) + else: + hosts = [] + return {host: _get_hostvars(host) for host in hosts} def _get_hosts_from_group(group): inventory = __context__["inventory"] + if group not in inventory: + return [] hosts = [host for host in inventory[group].get("hosts", [])] for child in inventory[group].get("children", []): - if child != "ungrouped": + child_info = _get_hosts_from_group(child) + if child_info not in hosts: hosts.extend(_get_hosts_from_group(child)) return hosts diff --git a/salt/states/ansiblegate.py b/salt/states/ansiblegate.py index 4afe6a020d..af5cb0f0e5 100644 --- a/salt/states/ansiblegate.py +++ b/salt/states/ansiblegate.py @@ -184,7 +184,11 @@ def playbooks(name, rundir=None, git_repo=None, git_kwargs=None, ansible_kwargs= checks = __salt__["ansible.playbooks"]( name, rundir=rundir, check=True, diff=True, **ansible_kwargs ) - if all( + if "stats" not in checks: + ret["comment"] = checks.get("stderr", checks) + ret["result"] = False + ret["changes"] = {} + elif all( not check["changed"] and not check["failures"] and not check["unreachable"] @@ -213,7 +217,11 @@ def playbooks(name, rundir=None, git_repo=None, git_kwargs=None, ansible_kwargs= results = __salt__["ansible.playbooks"]( name, rundir=rundir, diff=True, **ansible_kwargs ) - if all( + if "stats" not in results: + ret["comment"] = results.get("stderr", results) + ret["result"] = False + ret["changes"] = {} + elif all( not check["changed"] and not check["failures"] and not check["unreachable"] diff --git a/salt/utils/ansible.py b/salt/utils/ansible.py new file mode 100644 index 0000000000..1e14037fd3 --- /dev/null +++ b/salt/utils/ansible.py @@ -0,0 +1,41 @@ +import logging +import os + +# Import Salt libs +import salt.utils.json +import salt.utils.path +import salt.utils.stringutils +import salt.modules.cmdmod +from salt.exceptions import CommandExecutionError + +__virtualname__ = "ansible" + +log = logging.getLogger(__name__) + + +def __virtual__(): # pylint: disable=expected-2-blank-lines-found-0 + if salt.utils.path.which("ansible-inventory"): + return __virtualname__ + return (False, "Install `ansible` to use inventory") + + +def targets(inventory="/etc/ansible/hosts", **kwargs): + """ + Return the targets from the ansible inventory_file + Default: /etc/salt/roster + """ + if not os.path.isfile(inventory): + raise CommandExecutionError("Inventory file not found: {}".format(inventory)) + + extra_cmd = [] + if "export" in kwargs: + extra_cmd.append("--export") + if "yaml" in kwargs: + extra_cmd.append("--yaml") + inv = salt.modules.cmdmod.run( + "ansible-inventory -i {} --list {}".format(inventory, " ".join(extra_cmd)) + ) + if kwargs.get("yaml", False): + return salt.utils.stringutils.to_str(inv) + else: + return salt.utils.json.loads(salt.utils.stringutils.to_str(inv)) diff --git a/tests/pytests/unit/modules/test_ansiblegate.py b/tests/pytests/unit/modules/test_ansiblegate.py index 44c9b12acb..f357133000 100644 --- a/tests/pytests/unit/modules/test_ansiblegate.py +++ b/tests/pytests/unit/modules/test_ansiblegate.py @@ -1,9 +1,15 @@ # Author: Bo Maryniuk +import os import pytest + +import salt.config +import salt.loader import salt.modules.ansiblegate as ansiblegate import salt.utils.json +import salt.utils.path from tests.support.mock import ANY, MagicMock, patch +from tests.support.runtests import RUNTIME_VARS pytestmark = [ pytest.mark.skip_on_windows(reason="Not supported on Windows"), @@ -12,7 +18,7 @@ pytestmark = [ @pytest.fixture def configure_loader_modules(): - return {ansiblegate: {}} + return {ansiblegate: {"__utils__": {}}} def test_ansible_module_help(): @@ -133,3 +139,94 @@ def test_ansible_playbooks_return_retcode(): ): ret = ansiblegate.playbooks("fake-playbook.yml") assert "retcode" in ret + + +def test_ansible_targets(): + """ + Test ansible.targets execution module function. + :return: + """ + ansible_inventory_ret = """ +{ + "_meta": { + "hostvars": { + "uyuni-stable-ansible-centos7-1.tf.local": { + "ansible_ssh_private_key_file": "/etc/ansible/my_ansible_private_key" + }, + "uyuni-stable-ansible-centos7-2.tf.local": { + "ansible_ssh_private_key_file": "/etc/ansible/my_ansible_private_key" + } + } + }, + "all": { + "children": [ + "ungrouped" + ] + }, + "ungrouped": { + "hosts": [ + "uyuni-stable-ansible-centos7-1.tf.local", + "uyuni-stable-ansible-centos7-2.tf.local" + ] + } +} + """ + ansible_inventory_mock = MagicMock(return_value=ansible_inventory_ret) + with patch("salt.utils.path.which", MagicMock(return_value=True)): + opts = salt.config.DEFAULT_MINION_OPTS.copy() + utils = salt.loader.utils(opts, whitelist=["ansible"]) + with patch("salt.modules.cmdmod.run", ansible_inventory_mock), patch.dict( + ansiblegate.__utils__, utils + ), patch("os.path.isfile", MagicMock(return_value=True)): + ret = ansiblegate.targets() + assert ansible_inventory_mock.call_args + assert "_meta" in ret + assert "uyuni-stable-ansible-centos7-1.tf.local" in ret["_meta"]["hostvars"] + assert ( + "ansible_ssh_private_key_file" + in ret["_meta"]["hostvars"]["uyuni-stable-ansible-centos7-1.tf.local"] + ) + assert "all" in ret + assert len(ret["ungrouped"]["hosts"]) == 2 + + +def test_ansible_discover_playbooks_single_path(): + playbooks_dir = os.path.join( + RUNTIME_VARS.TESTS_DIR, "unit/files/playbooks/example_playbooks/" + ) + ret = ansiblegate.discover_playbooks(playbooks_dir) + assert playbooks_dir in ret + assert ret[playbooks_dir]["playbook1.yml"] == { + "fullpath": os.path.join(playbooks_dir, "playbook1.yml") + } + assert ret[playbooks_dir]["example-playbook2/site.yml"] == { + "fullpath": os.path.join(playbooks_dir, "example-playbook2/site.yml"), + "custom_inventory": os.path.join(playbooks_dir, "example-playbook2/hosts"), + } + + +def test_ansible_discover_playbooks_single_path_using_parameters(): + playbooks_dir = os.path.join( + RUNTIME_VARS.TESTS_DIR, "unit/files/playbooks/example_playbooks/" + ) + ret = ansiblegate.discover_playbooks( + playbooks_dir, playbook_extension="foobar", hosts_filename="deadbeaf" + ) + assert playbooks_dir in ret + assert ret[playbooks_dir] == {} + + +def test_ansible_discover_playbooks_multiple_locations(): + playbooks_dir = os.path.join( + RUNTIME_VARS.TESTS_DIR, "unit/files/playbooks/example_playbooks/" + ) + ret = ansiblegate.discover_playbooks(locations=[playbooks_dir, "/tmp/foobar"]) + assert playbooks_dir in ret + assert "/tmp/foobar" in ret + assert ret[playbooks_dir]["playbook1.yml"] == { + "fullpath": os.path.join(playbooks_dir, "playbook1.yml") + } + assert ret[playbooks_dir]["example-playbook2/site.yml"] == { + "fullpath": os.path.join(playbooks_dir, "example-playbook2/site.yml"), + "custom_inventory": os.path.join(playbooks_dir, "example-playbook2/hosts"), + } diff --git a/tests/unit/files/playbooks/example_playbooks/example-playbook2/hosts b/tests/unit/files/playbooks/example_playbooks/example-playbook2/hosts new file mode 100644 index 0000000000..75783285f6 --- /dev/null +++ b/tests/unit/files/playbooks/example_playbooks/example-playbook2/hosts @@ -0,0 +1,7 @@ +[databases] +host1 +host2 + +[webservers] +host3 +host4 diff --git a/tests/unit/files/playbooks/example_playbooks/example-playbook2/site.yml b/tests/unit/files/playbooks/example_playbooks/example-playbook2/site.yml new file mode 100644 index 0000000000..a64ebd5e18 --- /dev/null +++ b/tests/unit/files/playbooks/example_playbooks/example-playbook2/site.yml @@ -0,0 +1,28 @@ +--- +- name: update web servers + hosts: webservers + remote_user: root + + tasks: + - name: ensure apache is at the latest version + yum: + name: httpd + state: latest + - name: write the apache config file + template: + src: /srv/httpd.j2 + dest: /etc/httpd.conf + +- name: update db servers + hosts: databases + remote_user: root + + tasks: + - name: ensure postgresql is at the latest version + yum: + name: postgresql + state: latest + - name: ensure that postgresql is started + service: + name: postgresql + state: started diff --git a/tests/unit/files/playbooks/example_playbooks/playbook1.yml b/tests/unit/files/playbooks/example_playbooks/playbook1.yml new file mode 100644 index 0000000000..e258a101e1 --- /dev/null +++ b/tests/unit/files/playbooks/example_playbooks/playbook1.yml @@ -0,0 +1,5 @@ +--- +- hosts: all + gather_facts: false + tasks: + - ping: diff --git a/tests/unit/roster/test_ansible.py b/tests/unit/roster/test_ansible.py index 7f1144454b..c4ab8b7639 100644 --- a/tests/unit/roster/test_ansible.py +++ b/tests/unit/roster/test_ansible.py @@ -63,7 +63,7 @@ class AnsibleRosterTestCase(TestCase, mixins.LoaderModuleMockMixin): opts = salt.config.master_config( os.path.join(RUNTIME_VARS.TMP_CONF_DIR, "master") ) - utils = salt.loader.utils(opts, whitelist=["json", "stringutils"]) + utils = salt.loader.utils(opts, whitelist=["json", "stringutils", "ansible"]) runner = salt.loader.runner(opts, utils=utils, whitelist=["salt"]) return {ansible: {"__utils__": utils, "__opts__": {}, "__runner__": runner}} -- 2.34.1