diff --git a/_lastrevision b/_lastrevision index e5f7c2a..463ecd2 100644 --- a/_lastrevision +++ b/_lastrevision @@ -1 +1 @@ -4d10df19ca9defdab8bacf1371c6d36037601a7f \ No newline at end of file +a94708ad2eba9aa15413d989ab3361b2c980589e \ No newline at end of file diff --git a/improvements-on-ansiblegate-module-354.patch b/improvements-on-ansiblegate-module-354.patch new file mode 100644 index 0000000..b56c907 --- /dev/null +++ b/improvements-on-ansiblegate-module-354.patch @@ -0,0 +1,521 @@ +From aa0f845e2bbc37332db04c583f475cfe25304db6 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Tue, 20 Apr 2021 11:01:26 +0100 +Subject: [PATCH] Improvements on "ansiblegate" module (#354) + +* Allow collecting Ansible Inventory from a minion + +* Prevent crashing if ansible-playbook doesn't return JSON + +* Add new 'ansible.discover_playbooks' method + +* Include custom inventory when discovering Ansible playbooks + +* Enhance 'ansible.discover_playbooks' to accept a list of locations + +* Remove unused constants from Ansible utils + +* Avoid string concatenation to calculate extra cmd args + +* Add unit test for ansible.targets + +* Improve Ansible roster targetting + +* Add tests for new ansiblegate module functions + +* Fix issue dealing with ungrouped targets on inventory + +* Enable ansible utils for ansible roster tests + +* Remove unnecessary code from Ansible utils + +* Fix pylint issue + +* Fix issue in documentation +--- + salt/modules/ansiblegate.py | 166 +++++++++++++++++- + salt/roster/ansible.py | 18 +- + salt/utils/ansible.py | 44 +++++ + .../pytests/unit/modules/test_ansiblegate.py | 94 +++++++++- + .../example_playbooks/example-playbook2/hosts | 7 + + .../example-playbook2/site.yml | 28 +++ + .../playbooks/example_playbooks/playbook1.yml | 5 + + tests/unit/roster/test_ansible.py | 2 +- + 8 files changed, 354 insertions(+), 10 deletions(-) + create mode 100644 salt/utils/ansible.py + create mode 100644 tests/unit/files/playbooks/example_playbooks/example-playbook2/hosts + create mode 100644 tests/unit/files/playbooks/example_playbooks/example-playbook2/site.yml + create mode 100644 tests/unit/files/playbooks/example_playbooks/playbook1.yml + +diff --git a/salt/modules/ansiblegate.py b/salt/modules/ansiblegate.py +index 5d4b986ec2..4f96607a07 100644 +--- a/salt/modules/ansiblegate.py ++++ b/salt/modules/ansiblegate.py +@@ -426,7 +426,171 @@ def playbooks( + } + ret = __salt__["cmd.run_all"](**cmd_kwargs) + log.debug("Ansible Playbook Return: %s", ret) +- retdata = json.loads(ret["stdout"]) ++ try: ++ retdata = json.loads(ret["stdout"]) ++ except ValueError: ++ retdata = ret + if "retcode" in ret: + __context__["retcode"] = retdata["retcode"] = ret["retcode"] + return retdata ++ ++ ++def targets(**kwargs): ++ """ ++ Return the inventory from an Ansible inventory_file ++ ++ :param inventory: ++ The inventory file to read the inventory from. Default: "/etc/ansible/hosts" ++ ++ :param yaml: ++ Return the inventory as yaml output. Default: False ++ ++ :param export: ++ Return inventory as export format. Default: False ++ ++ CLI Example: ++ ++ .. code-block:: bash ++ ++ salt 'ansiblehost' ansible.targets ++ salt 'ansiblehost' ansible.targets inventory=my_custom_inventory ++ ++ """ ++ return __utils__["ansible.targets"](**kwargs) ++ ++ ++def discover_playbooks(path=None, ++ locations=None, ++ playbook_extension=None, ++ hosts_filename=None, ++ syntax_check=False): ++ """ ++ Discover Ansible playbooks stored under the given path or from multiple paths (locations) ++ ++ This will search for files matching with the playbook file extension under the given ++ root path and will also look for files inside the first level of directories in this path. ++ ++ The return of this function would be a dict like this: ++ ++ .. code-block:: python ++ ++ { ++ "/home/foobar/": { ++ "my_ansible_playbook.yml": { ++ "fullpath": "/home/foobar/playbooks/my_ansible_playbook.yml", ++ "custom_inventory": "/home/foobar/playbooks/hosts" ++ }, ++ "another_playbook.yml": { ++ "fullpath": "/home/foobar/playbooks/another_playbook.yml", ++ "custom_inventory": "/home/foobar/playbooks/hosts" ++ }, ++ "lamp_simple/site.yml": { ++ "fullpath": "/home/foobar/playbooks/lamp_simple/site.yml", ++ "custom_inventory": "/home/foobar/playbooks/lamp_simple/hosts" ++ }, ++ "lamp_proxy/site.yml": { ++ "fullpath": "/home/foobar/playbooks/lamp_proxy/site.yml", ++ "custom_inventory": "/home/foobar/playbooks/lamp_proxy/hosts" ++ } ++ }, ++ "/srv/playbooks/": { ++ "example_playbook/example.yml": { ++ "fullpath": "/srv/playbooks/example_playbook/example.yml", ++ "custom_inventory": "/srv/playbooks/example_playbook/hosts" ++ } ++ } ++ } ++ ++ :param path: ++ Path to discover playbooks from. ++ ++ :param locations: ++ List of paths to discover playbooks from. ++ ++ :param playbook_extension: ++ File extension of playbooks file to search for. Default: "yml" ++ ++ :param hosts_filename: ++ Filename of custom playbook inventory to search for. Default: "hosts" ++ ++ :param syntax_check: ++ Skip playbooks that do not pass "ansible-playbook --syntax-check" validation. Default: False ++ ++ :return: ++ The discovered playbooks under the given paths ++ ++ CLI Example: ++ ++ .. code-block:: bash ++ ++ salt 'ansiblehost' ansible.discover_playbooks path=/srv/playbooks/ ++ salt 'ansiblehost' ansible.discover_playbooks locations='["/srv/playbooks/", "/srv/foobar"]' ++ ++ """ ++ ++ if not path and not locations: ++ raise CommandExecutionError("You have to specify either 'path' or 'locations' arguments") ++ ++ if path and locations: ++ raise CommandExecutionError("You cannot specify 'path' and 'locations' at the same time") ++ ++ if not playbook_extension: ++ playbook_extension = "yml" ++ if not hosts_filename: ++ hosts_filename = "hosts" ++ ++ if path: ++ if not os.path.isabs(path): ++ raise CommandExecutionError("The given path is not an absolute path: {}".format(path)) ++ if not os.path.isdir(path): ++ raise CommandExecutionError("The given path is not a directory: {}".format(path)) ++ return {path: _explore_path(path, playbook_extension, hosts_filename, syntax_check)} ++ ++ if locations: ++ all_ret = {} ++ for location in locations: ++ all_ret[location] = _explore_path(location, playbook_extension, hosts_filename, syntax_check) ++ return all_ret ++ ++ ++def _explore_path(path, playbook_extension, hosts_filename, syntax_check): ++ ret = {} ++ ++ if not os.path.isabs(path): ++ log.error("The given path is not an absolute path: {}".format(path)) ++ return ret ++ if not os.path.isdir(path): ++ log.error("The given path is not a directory: {}".format(path)) ++ return ret ++ ++ try: ++ # Check files in the given path ++ for _f in os.listdir(path): ++ _path = os.path.join(path, _f) ++ if os.path.isfile(_path) and _path.endswith("." + playbook_extension): ++ ret[_f] = {"fullpath": _path} ++ # Check for custom inventory file ++ if os.path.isfile(os.path.join(path, hosts_filename)): ++ ret[_f].update({"custom_inventory": os.path.join(path, hosts_filename)}) ++ elif os.path.isdir(_path): ++ # Check files in the 1st level of subdirectories ++ for _f2 in os.listdir(_path): ++ _path2 = os.path.join(_path, _f2) ++ if os.path.isfile(_path2) and _path2.endswith("." + playbook_extension): ++ ret[os.path.join(_f, _f2)] = {"fullpath": _path2} ++ # Check for custom inventory file ++ if os.path.isfile(os.path.join(_path, hosts_filename)): ++ ret[os.path.join(_f, _f2)].update({"custom_inventory": os.path.join(_path, hosts_filename)}) ++ except Exception as exc: ++ raise CommandExecutionError("There was an exception while discovering playbooks: {}".format(exc)) ++ ++ # Run syntax check validation ++ if syntax_check: ++ check_command = ["ansible-playbook", "--syntax-check"] ++ try: ++ for pb in list(ret): ++ if __salt__["cmd.retcode"](check_command + [ret[pb]]): ++ del ret[pb] ++ except Exception as exc: ++ raise CommandExecutionError("There was an exception while checking syntax of playbooks: {}".format(exc)) ++ return ret +diff --git a/salt/roster/ansible.py b/salt/roster/ansible.py +index f17316bdd7..cc61f6fb7d 100644 +--- a/salt/roster/ansible.py ++++ b/salt/roster/ansible.py +@@ -89,7 +89,6 @@ Any of the [groups] or direct hostnames will return. The 'all' is special, and + """ + # Import Python libs + from __future__ import absolute_import, print_function, unicode_literals +- + import copy + import fnmatch + +@@ -121,27 +120,32 @@ def targets(tgt, tgt_type="glob", **kwargs): + Return the targets from the ansible inventory_file + Default: /etc/salt/roster + """ +- inventory = __runner__["salt.cmd"]( +- "cmd.run", "ansible-inventory -i {0} --list".format(get_roster_file(__opts__)) +- ) +- __context__["inventory"] = __utils__["json.loads"]( +- __utils__["stringutils.to_str"](inventory) ++ __context__["inventory"] = __utils__["ansible.targets"]( ++ inventory=get_roster_file(__opts__), **kwargs + ) + + if tgt_type == "glob": + hosts = [ + host for host in _get_hosts_from_group("all") if fnmatch.fnmatch(host, tgt) + ] ++ elif tgt_type == "list": ++ hosts = [host for host in _get_hosts_from_group("all") if host in tgt] + elif tgt_type == "nodegroup": + hosts = _get_hosts_from_group(tgt) ++ else: ++ hosts = [] ++ + return {host: _get_hostvars(host) for host in hosts} + + + def _get_hosts_from_group(group): + inventory = __context__["inventory"] ++ if group not in inventory: ++ return [] + hosts = [host for host in inventory[group].get("hosts", [])] + for child in inventory[group].get("children", []): +- if child != "ungrouped": ++ child_info = _get_hosts_from_group(child) ++ if child_info not in hosts: + hosts.extend(_get_hosts_from_group(child)) + return hosts + +diff --git a/salt/utils/ansible.py b/salt/utils/ansible.py +new file mode 100644 +index 0000000000..ee85cb656c +--- /dev/null ++++ b/salt/utils/ansible.py +@@ -0,0 +1,44 @@ ++# -*- coding: utf-8 -*- ++# Import Python libs ++from __future__ import absolute_import, print_function, unicode_literals ++import logging ++import os ++ ++# Import Salt libs ++import salt.utils.json ++import salt.utils.path ++import salt.utils.stringutils ++import salt.modules.cmdmod ++from salt.exceptions import CommandExecutionError ++ ++__virtualname__ = "ansible" ++ ++log = logging.getLogger(__name__) ++ ++ ++def __virtual__(): # pylint: disable=expected-2-blank-lines-found-0 ++ if salt.utils.path.which("ansible-inventory"): ++ return __virtualname__ ++ return (False, "Install `ansible` to use inventory") ++ ++ ++def targets(inventory="/etc/ansible/hosts", **kwargs): ++ """ ++ Return the targets from the ansible inventory_file ++ Default: /etc/salt/roster ++ """ ++ if not os.path.isfile(inventory): ++ raise CommandExecutionError("Inventory file not found: {}".format(inventory)) ++ ++ extra_cmd = [] ++ if "export" in kwargs: ++ extra_cmd.append("--export") ++ if "yaml" in kwargs: ++ extra_cmd.append("--yaml") ++ inv = salt.modules.cmdmod.run( ++ "ansible-inventory -i {} --list {}".format(inventory, " ".join(extra_cmd)) ++ ) ++ if kwargs.get("yaml", False): ++ return salt.utils.stringutils.to_str(inv) ++ else: ++ return salt.utils.json.loads(salt.utils.stringutils.to_str(inv)) +diff --git a/tests/pytests/unit/modules/test_ansiblegate.py b/tests/pytests/unit/modules/test_ansiblegate.py +index 42c0968a6e..24c7e5e6b3 100644 +--- a/tests/pytests/unit/modules/test_ansiblegate.py ++++ b/tests/pytests/unit/modules/test_ansiblegate.py +@@ -8,8 +8,11 @@ import pytest + import salt.modules.ansiblegate as ansible + import salt.utils.path + import salt.utils.platform ++import salt.config ++import salt.loader + from salt.exceptions import LoaderError + from tests.support.mock import MagicMock, MockTimedProc, patch ++from tests.support.runtests import RUNTIME_VARS + + pytestmark = pytest.mark.skipif( + salt.utils.platform.is_windows(), reason="Not supported on Windows" +@@ -18,7 +21,7 @@ pytestmark = pytest.mark.skipif( + + @pytest.fixture + def configure_loader_modules(): +- return {ansible: {}} ++ return {ansible: {"__utils__": {}}} + + + @pytest.fixture +@@ -201,3 +204,92 @@ def test_ansible_playbooks_return_retcode(resolver): + ): + ret = ansible.playbooks("fake-playbook.yml") + assert "retcode" in ret ++ ++ ++def test_ansible_targets(): ++ """ ++ Test ansible.targets execution module function. ++ :return: ++ """ ++ ansible_inventory_ret = """ ++{ ++ "_meta": { ++ "hostvars": { ++ "uyuni-stable-ansible-centos7-1.tf.local": { ++ "ansible_ssh_private_key_file": "/etc/ansible/my_ansible_private_key" ++ }, ++ "uyuni-stable-ansible-centos7-2.tf.local": { ++ "ansible_ssh_private_key_file": "/etc/ansible/my_ansible_private_key" ++ } ++ } ++ }, ++ "all": { ++ "children": [ ++ "ungrouped" ++ ] ++ }, ++ "ungrouped": { ++ "hosts": [ ++ "uyuni-stable-ansible-centos7-1.tf.local", ++ "uyuni-stable-ansible-centos7-2.tf.local" ++ ] ++ } ++} ++ """ ++ ansible_inventory_mock = MagicMock(return_value=ansible_inventory_ret) ++ with patch("salt.utils.path.which", MagicMock(return_value=True)): ++ opts = salt.config.DEFAULT_MINION_OPTS.copy() ++ utils = salt.loader.utils(opts, whitelist=["ansible"]) ++ with patch("salt.modules.cmdmod.run", ansible_inventory_mock), patch.dict( ++ ansible.__utils__, utils), patch( ++ "os.path.isfile", MagicMock(return_value=True) ++ ): ++ ret = ansible.targets() ++ assert ansible_inventory_mock.call_args ++ assert "_meta" in ret ++ assert "uyuni-stable-ansible-centos7-1.tf.local" in ret["_meta"]["hostvars"] ++ assert "ansible_ssh_private_key_file" in ret["_meta"]["hostvars"]["uyuni-stable-ansible-centos7-1.tf.local"] ++ assert "all" in ret ++ assert len(ret["ungrouped"]["hosts"]) == 2 ++ ++ ++def test_ansible_discover_playbooks_single_path(): ++ playbooks_dir = os.path.join( ++ RUNTIME_VARS.TESTS_DIR, "unit/files/playbooks/example_playbooks/" ++ ) ++ ret = ansible.discover_playbooks(playbooks_dir) ++ assert playbooks_dir in ret ++ assert ret[playbooks_dir]["playbook1.yml"] == { ++ "fullpath": os.path.join(playbooks_dir, "playbook1.yml") ++ } ++ assert ret[playbooks_dir]["example-playbook2/site.yml"] == { ++ "fullpath": os.path.join(playbooks_dir, "example-playbook2/site.yml"), ++ "custom_inventory": os.path.join(playbooks_dir, "example-playbook2/hosts"), ++ } ++ ++ ++def test_ansible_discover_playbooks_single_path_using_parameters(): ++ playbooks_dir = os.path.join( ++ RUNTIME_VARS.TESTS_DIR, "unit/files/playbooks/example_playbooks/" ++ ) ++ ret = ansible.discover_playbooks( ++ playbooks_dir, playbook_extension="foobar", hosts_filename="deadbeaf" ++ ) ++ assert playbooks_dir in ret ++ assert ret[playbooks_dir] == {} ++ ++ ++def test_ansible_discover_playbooks_multiple_locations(): ++ playbooks_dir = os.path.join( ++ RUNTIME_VARS.TESTS_DIR, "unit/files/playbooks/example_playbooks/" ++ ) ++ ret = ansible.discover_playbooks(locations=[playbooks_dir, "/tmp/foobar"]) ++ assert playbooks_dir in ret ++ assert "/tmp/foobar" in ret ++ assert ret[playbooks_dir]["playbook1.yml"] == { ++ "fullpath": os.path.join(playbooks_dir, "playbook1.yml") ++ } ++ assert ret[playbooks_dir]["example-playbook2/site.yml"] == { ++ "fullpath": os.path.join(playbooks_dir, "example-playbook2/site.yml"), ++ "custom_inventory": os.path.join(playbooks_dir, "example-playbook2/hosts"), ++ } +diff --git a/tests/unit/files/playbooks/example_playbooks/example-playbook2/hosts b/tests/unit/files/playbooks/example_playbooks/example-playbook2/hosts +new file mode 100644 +index 0000000000..75783285f6 +--- /dev/null ++++ b/tests/unit/files/playbooks/example_playbooks/example-playbook2/hosts +@@ -0,0 +1,7 @@ ++[databases] ++host1 ++host2 ++ ++[webservers] ++host3 ++host4 +diff --git a/tests/unit/files/playbooks/example_playbooks/example-playbook2/site.yml b/tests/unit/files/playbooks/example_playbooks/example-playbook2/site.yml +new file mode 100644 +index 0000000000..a64ebd5e18 +--- /dev/null ++++ b/tests/unit/files/playbooks/example_playbooks/example-playbook2/site.yml +@@ -0,0 +1,28 @@ ++--- ++- name: update web servers ++ hosts: webservers ++ remote_user: root ++ ++ tasks: ++ - name: ensure apache is at the latest version ++ yum: ++ name: httpd ++ state: latest ++ - name: write the apache config file ++ template: ++ src: /srv/httpd.j2 ++ dest: /etc/httpd.conf ++ ++- name: update db servers ++ hosts: databases ++ remote_user: root ++ ++ tasks: ++ - name: ensure postgresql is at the latest version ++ yum: ++ name: postgresql ++ state: latest ++ - name: ensure that postgresql is started ++ service: ++ name: postgresql ++ state: started +diff --git a/tests/unit/files/playbooks/example_playbooks/playbook1.yml b/tests/unit/files/playbooks/example_playbooks/playbook1.yml +new file mode 100644 +index 0000000000..e258a101e1 +--- /dev/null ++++ b/tests/unit/files/playbooks/example_playbooks/playbook1.yml +@@ -0,0 +1,5 @@ ++--- ++- hosts: all ++ gather_facts: false ++ tasks: ++ - ping: +diff --git a/tests/unit/roster/test_ansible.py b/tests/unit/roster/test_ansible.py +index a5cdcbbdbc..8bc9c1c6f7 100644 +--- a/tests/unit/roster/test_ansible.py ++++ b/tests/unit/roster/test_ansible.py +@@ -71,7 +71,7 @@ class AnsibleRosterTestCase(TestCase, mixins.LoaderModuleMockMixin): + opts = salt.config.master_config( + os.path.join(RUNTIME_VARS.TMP_CONF_DIR, "master") + ) +- utils = salt.loader.utils(opts, whitelist=["json", "stringutils"]) ++ utils = salt.loader.utils(opts, whitelist=["json", "stringutils", "ansible"]) + runner = salt.loader.runner(opts, utils=utils, whitelist=["salt"]) + return {ansible: {"__utils__": utils, "__opts__": {}, "__runner__": runner}} + +-- +2.31.1 + + diff --git a/salt.changes b/salt.changes index 16230d4..a527fb0 100644 --- a/salt.changes +++ b/salt.changes @@ -1,3 +1,13 @@ +------------------------------------------------------------------- +Tue Apr 20 12:18:06 UTC 2021 - Pablo Suárez Hernández + +- Improvements on "ansiblegate" module: + * New methods: ansible.targets / ansible.discover_playbooks + * General bugfixes + +- Added: + * improvements-on-ansiblegate-module-354.patch + ------------------------------------------------------------------- Tue Apr 13 15:03:48 UTC 2021 - Pablo Suárez Hernández diff --git a/salt.spec b/salt.spec index 7b1cd54..7e8d74d 100644 --- a/salt.spec +++ b/salt.spec @@ -394,6 +394,8 @@ Patch165: update-target-fix-for-salt-ssh-to-process-targets-li.patch Patch166: add-alibaba-cloud-linux-2-by-backporting-upstream-s-.patch # PATCH-FIX_OPENSUSE: https://github.com/openSUSE/salt/pull/353 Patch167: regression-fix-of-salt-ssh-on-processing-targets-353.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/60056 +Patch168: improvements-on-ansiblegate-module-354.patch BuildRoot: %{_tmppath}/%{name}-%{version}-build BuildRequires: logrotate @@ -956,6 +958,7 @@ cp %{S:5} ./.travis.yml %patch165 -p1 %patch166 -p1 %patch167 -p1 +%patch168 -p1 %build # Putting /usr/bin at the front of $PATH is needed for RHEL/RES 7. Without this